Skip to content

SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.

Module clean-ups, tons of bug fixes and tweaks.

smicallef38985a6dbec4
diff --git a/modules/sfp__stor_stdout.py b/modules/sfp__stor_stdout.py
index b9a323aa..5f193b9a 100644
--- a/modules/sfp__stor_stdout.py
+++ b/modules/sfp__stor_stdout.py
@@ -85,7 +85,6 @@ class sfp__stor_stdout(SpiderFootPlugin):
d['type'] = self.opts['_eventtypes'][event.eventType]
print(json.dumps(d))
-
# Handle events sent to this module
def handleEvent(self, sfEvent):
if sfEvent.eventType == "ROOT":
diff --git a/modules/sfp_accounts.py b/modules/sfp_accounts.py
index acbb6c8f..ebd83ab9 100644
--- a/modules/sfp_accounts.py
+++ b/modules/sfp_accounts.py
@@ -157,7 +157,7 @@ class sfp_accounts(SpiderFootPlugin):
self.sf.info("Spawning thread to check site: " + site['name'] + \
" / " + site['check_uri'].format(account=name))
- t.append(threading.Thread(name='sfp_accounts_' + site['name'],
+ t.append(threading.Thread(name='thread_sfp_accounts_' + site['name'],
target=self.checkSite, args=(name, site)))
t[i].start()
i += 1
@@ -166,7 +166,7 @@ class sfp_accounts(SpiderFootPlugin):
while running:
found = False
for rt in threading.enumerate():
- if rt.name.startswith("sfp_accounts_"):
+ if rt.name.startswith("thread_sfp_accounts_"):
found = True
if not found:
diff --git a/modules/sfp_adblock.py b/modules/sfp_adblock.py
index 2f2e1d35..ddcb8bfd 100644
--- a/modules/sfp_adblock.py
+++ b/modules/sfp_adblock.py
@@ -27,13 +27,13 @@ class sfp_adblock(SpiderFootPlugin):
"blocklist": "AdBlockPlus block list."
}
- results = list()
+ results = None
rules = None
errorState = False
def setup(self, sfc, userOpts=dict()):
self.sf = sfc
- self.results = list()
+ self.results = self.tempStorage()
self.rules = None
self.errorState = False
@@ -81,7 +81,7 @@ class sfp_adblock(SpiderFootPlugin):
pagetype = "_INTERNAL"
if eventData not in self.results:
- self.results.append(eventData)
+ self.results[eventData] = True
else:
self.sf.debug("Already checked this page for AdBlock matching, skipping.")
return None
diff --git a/modules/sfp_ahmia.py b/modules/sfp_ahmia.py
index c7a6b9a1..9f4db495 100644
--- a/modules/sfp_ahmia.py
+++ b/modules/sfp_ahmia.py
@@ -72,14 +72,10 @@ class sfp_ahmia(SpiderFootPlugin):
if self.checkForStop():
return None
- # Submit the search results for analysis
- evt = SpiderFootEvent("SEARCH_ENGINE_WEB_CONTENT", data['content'],
- self.__name__, event)
- self.notifyListeners(evt)
-
links = re.findall("redirect_url=(.[^\"]+)\"",
data['content'], re.IGNORECASE | re.DOTALL)
+ reported = False
for link in links:
if link in self.results:
continue
@@ -110,14 +106,21 @@ class sfp_ahmia(SpiderFootPlugin):
self.sf.debug("String not found in content.")
continue
- data = res['content'][startIndex:endIndex]
- evt = SpiderFootEvent("DARKNET_MENTION_CONTENT", "..." + data + "...",
+ wdata = res['content'][startIndex:endIndex]
+ evt = SpiderFootEvent("DARKNET_MENTION_CONTENT", "..." + wdata + "...",
self.__name__, evt)
self.notifyListeners(evt)
-
+ reported = True
else:
evt = SpiderFootEvent("DARKNET_MENTION_URL", link, self.__name__, event)
self.notifyListeners(evt)
+ reported = True
+
+ if reported:
+ # Submit the search results for analysis
+ evt = SpiderFootEvent("SEARCH_ENGINE_WEB_CONTENT", data['content'],
+ self.__name__, event)
+ self.notifyListeners(evt)
# End of sfp_ahmia class
diff --git a/modules/sfp_archiveorg.py b/modules/sfp_archiveorg.py
index 33b436cf..b4b57cc4 100644
--- a/modules/sfp_archiveorg.py
+++ b/modules/sfp_archiveorg.py
@@ -47,12 +47,12 @@ class sfp_archiveorg(SpiderFootPlugin):
"javascriptpages": "Query the Wayback Machine for historic versions of URLs using Javascript."
}
- results = list()
+ results = None
foundDates = list()
def setup(self, sfc, userOpts=dict()):
self.sf = sfc
- self.results = list()
+ self.results = self.tempStorage()
self.foundDates = list()
for opt in userOpts.keys():
@@ -104,7 +104,7 @@ class sfp_archiveorg(SpiderFootPlugin):
if eventData in self.results:
return None
else:
- self.results.append(eventData)
+ self.results[eventData] = True
for daysback in self.opts['farback'].split(","):
newDate = datetime.datetime.now() - datetime.timedelta(days=int(daysback))
diff --git a/modules/sfp_azureblobstorage.py b/modules/sfp_azureblobstorage.py
index 4d299e42..36d95f14 100644
--- a/modules/sfp_azureblobstorage.py
+++ b/modules/sfp_azureblobstorage.py
@@ -32,7 +32,6 @@ class sfp_azureblobstorage(SpiderFootPlugin):
results = None
s3results = None
- lock = None
def setup(self, sfc, userOpts=dict()):
self.sf = sfc
@@ -72,7 +71,7 @@ class sfp_azureblobstorage(SpiderFootPlugin):
return None
self.sf.info("Spawning thread to check bucket: " + site)
- t.append(threading.Thread(name='sfp_azureblobstorages_' + site,
+ t.append(threading.Thread(name='thread_sfp_azureblobstorages_' + site,
target=self.checkSite, args=(site,)))
t[i].start()
i += 1
@@ -81,7 +80,7 @@ class sfp_azureblobstorage(SpiderFootPlugin):
while running:
found = False
for rt in threading.enumerate():
- if rt.name.startswith("sfp_azureblobstorages_"):
+ if rt.name.startswith("thread_sfp_azureblobstorages_"):
found = True
if not found:
diff --git a/modules/sfp_bitcoin.py b/modules/sfp_bitcoin.py
index bbf594b5..5e9ff2b0 100644
--- a/modules/sfp_bitcoin.py
+++ b/modules/sfp_bitcoin.py
@@ -11,10 +11,7 @@
# Licence: GPL
# -------------------------------------------------------------------------------
-try:
- import re2 as re
-except ImportError:
- import re
+import re
from hashlib import sha256
from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
@@ -24,11 +21,11 @@ class sfp_bitcoin(SpiderFootPlugin):
# Default options
opts = {}
- results = list()
+ results = None
def setup(self, sfc, userOpts=dict()):
self.sf = sfc
- self.results = list()
+ self.results = self.tempStorage()
for opt in userOpts.keys():
self.opts[opt] = userOpts[opt]
@@ -69,7 +66,7 @@ class sfp_bitcoin(SpiderFootPlugin):
if sourceData in self.results:
return None
else:
- self.results.append(sourceData)
+ self.results[sourceData] = True
self.sf.debug("Received event, " + eventName + ", from " + srcModuleName)
diff --git a/modules/sfp_company.py b/modules/sfp_company.py
index 49c964cc..bf1d8a46 100644
--- a/modules/sfp_company.py
+++ b/modules/sfp_company.py
@@ -11,17 +11,12 @@
# Licence: GPL
# -------------------------------------------------------------------------------
-try:
- import re2 as re
-except ImportError:
- import re
-
+import re
from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
class sfp_company(SpiderFootPlugin):
"""Company Names:Footprint,Investigate,Passive:Content Analysis::Identify company names in any obtained data."""
-
# Default options
opts = {
# options specific to this module
@@ -85,7 +80,7 @@ class sfp_company(SpiderFootPlugin):
return None
if eventName == "TARGET_WEB_CONTENT":
- url = event.sourceEvent.data
+ url = event.actualSource
if self.opts['filterjscss'] and (".js" in url or ".css" in url):
self.sf.debug("Ignoring web content from CSS/JS.")
return None
diff --git a/modules/sfp_cookie.py b/modules/sfp_cookie.py
index 3958c136..401f4435 100644
--- a/modules/sfp_cookie.py
+++ b/modules/sfp_cookie.py
@@ -10,9 +10,9 @@
# Licence: GPL
# -------------------------------------------------------------------------------
+import json
from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
-
class sfp_cookie(SpiderFootPlugin):
"""Cookies:Footprint,Investigate,Passive:Crawling and Scanning::Extract Cookies from HTTP headers."""
@@ -43,8 +43,7 @@ class sfp_cookie(SpiderFootPlugin):
eventName = event.eventType
srcModuleName = event.module
eventData = event.data
- parentEvent = event.sourceEvent
- eventSource = event.sourceEvent.data
+ eventSource = event.actualSource
self.sf.debug("Received event, " + eventName + ", from " + srcModuleName)
if eventSource in self.results:
@@ -56,9 +55,17 @@ class sfp_cookie(SpiderFootPlugin):
self.sf.debug("Not collecting cookies from external sites.")
return None
- if 'set-cookie' in eventData:
- evt = SpiderFootEvent("TARGET_WEB_COOKIE", eventData['set-cookie'],
- self.__name__, parentEvent)
+ try:
+ jdata = json.loads(eventData)
+ if jdata == None:
+ return None
+ except BaseException as e:
+ self.sf.error("Received HTTP headers from another module in an unexpected format.", False)
+ return None
+
+ if 'set-cookie' in jdata:
+ evt = SpiderFootEvent("TARGET_WEB_COOKIE", jdata['set-cookie'],
+ self.__name__, event)
self.notifyListeners(evt)
# End of sfp_cookie class
diff --git a/modules/sfp_crossref.py b/modules/sfp_crossref.py
index fa3035b3..0de2ea19 100644
--- a/modules/sfp_crossref.py
+++ b/modules/sfp_crossref.py
@@ -16,11 +16,9 @@
import re
from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
-
class sfp_crossref(SpiderFootPlugin):
"""Cross-Reference:Footprint:Crawling and Scanning::Identify whether other domains are associated ('Affiliates') of the target."""
-
# Default options
opts = {
'checkbase': True
@@ -32,21 +30,18 @@ class sfp_crossref(SpiderFootPlugin):
}
# Internal results tracking
- results = None
- fetched = list()
+ fetched = None
def setup(self, sfc, userOpts=dict()):
self.sf = sfc
- self.results = self.tempStorage()
- self.fetched = list()
+ self.fetched = self.tempStorage()
for opt in userOpts.keys():
self.opts[opt] = userOpts[opt]
# What events is this module interested in for input
def watchedEvents(self):
- return ['LINKED_URL_EXTERNAL', 'SIMILARDOMAIN',
- 'CO_HOSTED_SITE', 'DARKNET_MENTION_URL']
+ return ['LINKED_URL_EXTERNAL', 'SIMILARDOMAIN', 'CO_HOSTED_SITE', 'DARKNET_MENTION_URL']
# What events this module produces
# This is to support the end user in selecting modules based on events
@@ -78,11 +73,11 @@ class sfp_crossref(SpiderFootPlugin):
self.sf.debug("Ignoring " + eventData + " as already tested")
return
else:
- self.fetched.append(eventData)
+ self.fetched[eventData] = True
self.sf.debug("Testing for affiliation: " + eventData)
res = self.sf.fetchUrl(eventData, timeout=self.opts['_fetchtimeout'],
- useragent=self.opts['_useragent'])
+ useragent=self.opts['_useragent'], sizeLimit=10000000)
if res['content'] is None:
self.sf.debug("Ignoring " + eventData + " as no data returned")
@@ -108,10 +103,11 @@ class sfp_crossref(SpiderFootPlugin):
if url in self.fetched:
return None
else:
- self.fetched.append(url)
+ self.fetched[url] = True
res = self.sf.fetchUrl(url, timeout=self.opts['_fetchtimeout'],
- useragent=self.opts['_useragent'])
+ useragent=self.opts['_useragent'],
+ sizeLimit=10000000)
if res['content'] is not None:
for name in self.getTarget().getNames():
pat = re.compile("([\.\'\/\"\ ]" + name + "[\'\/\"\ ])",
@@ -134,4 +130,3 @@ class sfp_crossref(SpiderFootPlugin):
evt2.moduleDataSource = event.moduleDataSource
self.notifyListeners(evt2)
-# End of sfp_crossref class
diff --git a/modules/sfp_digitaloceanspace.py b/modules/sfp_digitaloceanspace.py
index c711a2a5..517f7586 100644
--- a/modules/sfp_digitaloceanspace.py
+++ b/modules/sfp_digitaloceanspace.py
@@ -33,7 +33,7 @@ class sfp_digitaloceanspace(SpiderFootPlugin):
}
results = None
- s3results = None
+ s3results = dict()
lock = None
def setup(self, sfc, userOpts=dict()):
@@ -56,16 +56,17 @@ class sfp_digitaloceanspace(SpiderFootPlugin):
return ["CLOUD_STORAGE_BUCKET", "CLOUD_STORAGE_BUCKET_OPEN"]
def checkSite(self, url):
- res = self.sf.fetchUrl(url, timeout=10, useragent="SpiderFoot", noLog=True)
+ res = self.sf.fetchUrl(url, timeout=10, useragent="SpiderFoot")
if res['code'] not in [ "301", "302", "200" ] and \
(res['content'] is None or "NoSuchBucket" in res['content']):
self.sf.debug("Not a valid bucket: " + url)
else:
- with self.lock:
- if "ListBucketResult" in res['content']:
+ if "ListBucketResult" in res['content']:
+ with self.lock:
self.s3results[url] = res['content'].count("<Key>")
- else:
+ else:
+ with self.lock:
self.s3results[url] = 0
def threadSites(self, siteList):
@@ -80,7 +81,7 @@ class sfp_digitaloceanspace(SpiderFootPlugin):
return None
self.sf.info("Spawning thread to check bucket: " + site)
- t.append(threading.Thread(name='sfp_digitaloceanspaces_' + site,
+ t.append(threading.Thread(name='thread_sfp_digitaloceanspaces_' + site,
target=self.checkSite, args=(site,)))
t[i].start()
i += 1
@@ -89,7 +90,7 @@ class sfp_digitaloceanspace(SpiderFootPlugin):
while running:
found = False
for rt in threading.enumerate():
- if rt.name.startswith("sfp_digitaloceanspaces_"):
+ if rt.name.startswith("thread_sfp_digitaloceanspaces_"):
found = True
if not found:
@@ -163,7 +164,8 @@ class sfp_digitaloceanspace(SpiderFootPlugin):
evt = SpiderFootEvent("CLOUD_STORAGE_BUCKET", bucket[0] + ":" + bucket[1], self.__name__, event)
self.notifyListeners(evt)
if bucket[2] != "0":
- evt = SpiderFootEvent("CLOUD_STORAGE_BUCKET_OPEN", bucket[0] + ":" + bucket[1] + ": " + bucket[2] + " files found.",
+ bucketname = bucket[1].replace("//", "")
+ evt = SpiderFootEvent("CLOUD_STORAGE_BUCKET_OPEN", bucketname + ":" + bucket[1] + ": " + bucket[2] + " files found.",
self.__name__, evt)
self.notifyListeners(evt)
diff --git a/modules/sfp_dnsbrute.py b/modules/sfp_dnsbrute.py
index 8e2b9ab3..017f82bb 100644
--- a/modules/sfp_dnsbrute.py
+++ b/modules/sfp_dnsbrute.py
@@ -29,30 +29,27 @@ class sfp_dnsbrute(SpiderFootPlugin):
"top10000": False,
"numbersuffix": True,
"numbersuffixlimit": True,
- "maxthreads": 20
+ "_maxthreads": 100
}
# Option descriptions
optdescs = {
- 'skipcommonwildcard': "If wildcard DNS is detected, don't attempt brute-forcing.",
+ 'skipcommonwildcard': "If wildcard DNS is detected, don't bother brute-forcing.",
'domainonly': "Only attempt to brute-force names on domain names, not hostnames (some hostnames are also sub-domains).",
'commons': "Try a list of about 750 common hostnames/sub-domains.",
'top10000': "Try a further 10,000 common hostnames/sub-domains. Will make the scan much slower.",
'numbersuffix': "For any host found, try appending 1, 01, 001, -1, -01, -001, 2, 02, etc. (up to 10)",
- 'numbersuffixlimit': "Limit using the number suffixes for hosts that have already been resolved? If disabled this will significantly extend the duration of scans.",
- 'maxthreads': "Maximum number of concurrent resolution attempts."
+ 'numbersuffixlimit': "Limit using the number suffixes for hosts that have already been resolved? If disabled this will significantly extend the duration of scans."
}
- events = dict()
- resolveCache = dict()
- sublist = dict()
+ events = None
+ sublist = None
lock = None
def setup(self, sfc, userOpts=dict()):
self.sf = sfc
- self.sublist = dict()
- self.events = dict()
- self.resolveCache = dict()
+ self.sublist = self.tempStorage()
+ self.events = self.tempStorage()
self.__dataSource__ = "DNS"
self.lock = threading.Lock()
@@ -117,7 +114,7 @@ class sfp_dnsbrute(SpiderFootPlugin):
# Spawn threads for scanning
self.sf.info("Spawning threads to check hosts: " + str(hostList))
for name in hostList:
- tn = 'sfp_dnsbrute_' + str(random.SystemRandom().randint(0, 999999999))
+ tn = 'thread_sfp_dnsbrute_' + str(random.randint(0, 999999999))
t.append(threading.Thread(name=tn, target=self.tryHost, args=(name,)))
t[i].start()
i += 1
@@ -126,13 +123,13 @@ class sfp_dnsbrute(SpiderFootPlugin):
while running:
found = False
for rt in threading.enumerate():
- if rt.name.startswith("sfp_dnsbrute_"):
+ if rt.name.startswith("thread_sfp_dnsbrute_"):
found = True
if not found:
running = False
- time.sleep(0.25)
+ time.sleep(0.05)
for res in self.hostResults:
if self.hostResults.get(res, False):
@@ -160,10 +157,14 @@ class sfp_dnsbrute(SpiderFootPlugin):
self.events[eventDataHash] = True
# Handle Unicode characters in the name
- if type(eventData) != unicode:
- eventData = unicode(eventData, "utf-8", errors='replace').encode("idna")
- else:
- eventData = eventData.encode("idna")
+ try:
+ if type(eventData) != unicode:
+ eventData = unicode(eventData, "utf-8", errors='replace').encode("idna")
+ else:
+ eventData = eventData.encode("idna")
+ except BaseException as e:
+ self.sf.debug("IDNA-parsing issue for " + eventData + ": " + str(e))
+ return None
if eventName == "INTERNET_NAME" and not self.getTarget().matches(eventData, includeChildren=False):
if not self.opts['numbersuffix']:
@@ -173,10 +174,11 @@ class sfp_dnsbrute(SpiderFootPlugin):
return None
h, dom = eventData.split(".", 1)
+
# Try resolving common names
wildcard = self.sf.checkDnsWildcard(dom)
if self.opts['skipcommonwildcard'] and wildcard:
- self.sf.debug("Wildcard DNS detected.")
+ self.sf.debug("Wildcard DNS detected on " + dom + " so skipping host iteration.")
return None
dom = "." + dom
@@ -212,7 +214,7 @@ class sfp_dnsbrute(SpiderFootPlugin):
name = sub + "." + eventData
- if len(targetList) <= self.opts['maxthreads']:
+ if len(targetList) <= self.opts['_maxthreads']:
targetList.append(name)
else:
self.tryHostWrapper(targetList, event)
@@ -237,7 +239,7 @@ class sfp_dnsbrute(SpiderFootPlugin):
nextsubs[s + "-0" + str(i) + dom] = True
nextsubs[s + "-00" + str(i) + dom] = True
- if len(nextsubs.keys()) >= self.opts['maxthreads']:
+ if len(nextsubs.keys()) >= self.opts['_maxthreads']:
self.tryHostWrapper(nextsubs.keys(), event)
nextsubs = dict()
diff --git a/modules/sfp_dnscommonsrv.py b/modules/sfp_dnscommonsrv.py
index 32b07522..47d04149 100644
--- a/modules/sfp_dnscommonsrv.py
+++ b/modules/sfp_dnscommonsrv.py
@@ -24,7 +24,7 @@ class sfp_dnscommonsrv(SpiderFootPlugin):
# Option descriptions
optdescs = {}
- events = dict()
+ events = None
commonsrv = [ # LDAP/Kerberos, used for Active Directory
# https://technet.microsoft.com/en-us/library/cc961719.aspx
@@ -67,7 +67,7 @@ class sfp_dnscommonsrv(SpiderFootPlugin):
def setup(self, sfc, userOpts=dict()):
self.sf = sfc
- self.events = dict()
+ self.events = self.tempStorage()
self.__dataSource__ = "DNS"
# What events is this module interested in for input
diff --git a/modules/sfp_dnsneighbor.py b/modules/sfp_dnsneighbor.py
index e7b30e2a..fa2aca49 100644
--- a/modules/sfp_dnsneighbor.py
+++ b/modules/sfp_dnsneighbor.py
@@ -11,6 +11,7 @@
# Licence: GPL
# -------------------------------------------------------------------------------
+import dns
from netaddr import IPAddress
from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
@@ -29,15 +30,15 @@ class sfp_dnsneighbor(SpiderFootPlugin):
'lookasidecount': "If look-aside is enabled, the number of IPs on each 'side' of the IP to look up"
}
- events = dict()
- domresults = dict()
- hostresults = dict()
+ events = None
+ domresults = None
+ hostresults = None
def setup(self, sfc, userOpts=dict()):
self.sf = sfc
- self.events = dict()
- self.domresults = dict()
- self.hostresults = dict()
+ self.events = self.tempStorage()
+ self.domresults = self.tempStorage()
+ self.hostresults = self.tempStorage()
self.__dataSource__ = "DNS"
for opt in userOpts.keys():
@@ -51,7 +52,7 @@ class sfp_dnsneighbor(SpiderFootPlugin):
# This is to support the end user in selecting modules based on events
# produced.
def producedEvents(self):
- return ["IP_ADDRESS", "AFFILIATE_IPADDR"]
+ return ["AFFILIATE_IPADDR", "IP_ADDRESS"]
# Handle events sent to this module
def handleEvent(self, event):
@@ -75,18 +76,22 @@ class sfp_dnsneighbor(SpiderFootPlugin):
self.sf.error("Invalid IP address received: " + eventData, False)
return None
- minip = IPAddress(int(ip) - self.opts['lookasidecount'])
- maxip = IPAddress(int(ip) + self.opts['lookasidecount'])
+ try:
+ minip = IPAddress(int(ip) - self.opts['lookasidecount'])
+ maxip = IPAddress(int(ip) + self.opts['lookasidecount'])
+ except BaseException as e:
+ self.sf.error("Received an invalid IP address: " + eventData, False)
+ return None
+
self.sf.debug("Lookaside max: " + str(maxip) + ", min: " + str(minip))
s = int(minip)
c = int(maxip)
while s <= c:
- if self.checkForStop():
- return None
-
sip = str(IPAddress(s))
self.sf.debug("Attempting look-aside lookup of: " + sip)
+ if self.checkForStop():
+ return None
if sip in self.hostresults or sip == eventData:
s += 1
@@ -134,17 +139,16 @@ class sfp_dnsneighbor(SpiderFootPlugin):
self.processHost(addr, parent, True)
s += 1
- # Process a host/IP, parentEvent is the event that represents this entity
def processHost(self, host, parentEvent, affiliate=None):
parentHash = self.sf.hashstring(parentEvent.data)
if host not in self.hostresults:
- self.hostresults[host] = list(parentHash)
+ self.hostresults[host] = [parentHash]
else:
if parentHash in self.hostresults[host] or parentEvent.data == host:
self.sf.debug("Skipping host, " + host + ", already processed.")
return None
else:
- self.hostresults[host].append(parentHash)
+ self.hostresults[host] = self.hostresults[host] + [parentHash]
self.sf.debug("Found host: " + host)
# If the returned hostname is aliaseed to our
@@ -172,6 +176,7 @@ class sfp_dnsneighbor(SpiderFootPlugin):
if self.sf.validIP(host):
htype = "IP_ADDRESS"
+ # If names were found, leave them to sfp_dnsresolve to resolve
if not htype:
return None
diff --git a/modules/sfp_dnsresolve.py b/modules/sfp_dnsresolve.py
index 58b8a8d6..7b13cda2 100644
--- a/modules/sfp_dnsresolve.py
+++ b/modules/sfp_dnsresolve.py
@@ -35,15 +35,15 @@ class sfp_dnsresolve(SpiderFootPlugin):
'maxnetblock': "Maximum owned netblock size to look up all IPs within (CIDR value, 24 = /24, 16 = /16, etc.)"
}
- events = dict()
- domresults = dict()
- hostresults = dict()
+ events = None
+ domresults = None
+ hostresults = None
def setup(self, sfc, userOpts=dict()):
self.sf = sfc
- self.events = dict()
- self.domresults = dict()
- self.hostresults = dict()
+ self.events = self.tempStorage()
+ self.domresults = self.tempStorage()
+ self.hostresults = self.tempStorage()
self.__dataSource__ = "DNS"
for opt in userOpts.keys():
@@ -52,31 +52,8 @@ class sfp_dnsresolve(SpiderFootPlugin):
def enrichTarget(self, target):
ret = list()
# If it's an IP, get the hostname it reverse resolves to
- if target.getType() == "IP_ADDRESS":
- ret = self.sf.resolveIP(target.getValue())
- if target.getType() == "INTERNET_NAME":
- ret = self.sf.resolveHost(target.getValue())
- if target.getType() == "NETBLOCK_OWNER":
- ret = list()
- for addr in IPNetwork(target.getValue()):
- ipaddr = str(addr)
- if ipaddr.split(".")[3] in ['255', '0']:
- continue
- if '255' in ipaddr.split("."):
- continue
- ret.append(ipaddr)
-
- # Add the reverse-resolved hostnames as aliases too..
- names = self.sf.resolveIP(ipaddr)
-
- if self.opts['validatereverse']:
- for host in names:
- chk = self.sf.resolveHost(host)
- if ipaddr in chk:
- ret.append(host)
- else:
- ret.extend(names)
-
+ self.sf.info("Identifying aliases for specified target(s)")
+ ret = self.sf.resolveTargets(target, self.opts['validatereverse'])
if not ret:
return target
@@ -88,6 +65,7 @@ class sfp_dnsresolve(SpiderFootPlugin):
idnahost = host.encode("idna")
if idnahost != host:
target.setAlias(idnahost, "INTERNET_NAME")
+
# If the target was a hostname/sub-domain, we can
# add the domain as an alias for the target. But
# not if the target was an IP or subnet.
@@ -95,7 +73,7 @@ class sfp_dnsresolve(SpiderFootPlugin):
# dom = self.sf.hostDomain(host, self.opts['_internettlds'])
# target.setAlias(dom, "INTERNET_NAME")
- self.sf.debug("Aliases identified: " + str(target.getAliases()))
+ self.sf.info("Aliases identified: " + str(target.getAliases()))
return target
@@ -104,7 +82,7 @@ class sfp_dnsresolve(SpiderFootPlugin):
return [
# Events that need some kind of DNS treatment
"CO_HOSTED_SITE", "AFFILIATE_INTERNET_NAME", "NETBLOCK_OWNER",
- "IP_ADDRESS", "INTERNET_NAME", "AFFILIATE_IPADDR",
+ "IP_ADDRESS", "INTERNET_NAME", "AFFILIATE_IPADDR",
# Events that may contain hostnames in their content
"TARGET_WEB_CONTENT", "BASE64_DATA", "AFFILIATE_DOMAIN_WHOIS",
"CO_HOSTED_SITE_DOMAIN_WHOIS", "DOMAN_WHOIS", "NETBLOCK_WHOIS",
@@ -119,10 +97,11 @@ class sfp_dnsresolve(SpiderFootPlugin):
# produced.
def producedEvents(self):
return ["IP_ADDRESS", "INTERNET_NAME", "AFFILIATE_INTERNET_NAME",
- "AFFILIATE_IPADDR", "DOMAIN_NAME", "IPV6_ADDRESS",
+ "AFFILIATE_IPADDR", "DOMAIN_NAME", "IPV6_ADDRESS",
"DOMAIN_NAME_PARENT", "CO_HOSTED_SITE_DOMAIN", "AFFILIATE_DOMAIN",
"INTERNET_NAME_UNRESOLVED"]
+
# Handle events sent to this module
def handleEvent(self, event):
eventName = event.eventType
@@ -132,8 +111,8 @@ class sfp_dnsresolve(SpiderFootPlugin):
addrs = None
parentEvent = event
- # Don't be recursive, and trust that sfp_dnsbrute knows what it's doing
- if srcModuleName in [ "sfp_dnsresolve", "sfp_dnsbrute" ]:
+ # Don't be recursive
+ if srcModuleName in [ "sfp_dnsresolve" ]:
return None
self.sf.debug("Received event, " + eventName + ", from " + srcModuleName)
@@ -157,24 +136,43 @@ class sfp_dnsresolve(SpiderFootPlugin):
# Search for IPs/hosts in raw data, but obviously nothing this module
# already produces, as those things are already entities, not raw data.
if eventName not in self.producedEvents():
- if type(eventData) in [dict, list]:
- eventDataStr = str(eventData)
- else:
- eventDataStr = eventData
- data = urllib2.unquote(eventDataStr)
+ data = urllib2.unquote(eventData)
for name in self.getTarget().getNames():
if self.checkForStop():
return None
+
+ offset = data.find(name)
+ if offset < 0:
+ continue
+
+ if offset == 0:
+ offset += len(name)
+
pat = re.compile("(%..)?([a-zA-Z0-9\-\.]+\." + name + ")", re.IGNORECASE)
- matches = re.findall(pat, data)
- if matches:
- for match in matches:
- # Wildcard certs will come in as .blah.blah
- if match[1].startswith("."):
- m = match[1][1:]
- else:
- m = match[1]
- self.processHost(m, parentEvent, False)
+ while offset >= 0:
+ offset = data.find(name, offset)
+ #print "found at offset: " + str(offset)
+ if offset < 0:
+ break
+
+ # Get 200 bytes before the name to try and get hostnames
+ chunkhost = data[(offset-200):(offset+len(name)+1)]
+
+ try:
+ matches = re.findall(pat, chunkhost)
+ if matches:
+ for match in matches:
+ # Wildcard certs will come in as .blah.blah
+ if match[1].startswith("."):
+ m = match[1][1:]
+ else:
+ m = match[1]
+ self.processHost(m, parentEvent, False)
+ except Exception as e:
+ self.error("Error applying regex to data (" + str(e) + ")", False)
+
+ offset += len(name)
+
# Nothing left to do with internal links and raw data
return None
@@ -204,9 +202,9 @@ class sfp_dnsresolve(SpiderFootPlugin):
if self.checkForStop():
return None
- addrs = self.sf.resolveIP(ipaddr)
- if len(addrs) > 0:
+ addrs = self.sf.resolveIP(ipaddr)
+ if addrs:
self.sf.debug("Found a reversed hostname from " + ipaddr +
" (" + str(addrs) + ")")
for addr in addrs:
@@ -250,13 +248,13 @@ class sfp_dnsresolve(SpiderFootPlugin):
def processHost(self, host, parentEvent, affiliate=None):
parentHash = self.sf.hashstring(parentEvent.data)
if host not in self.hostresults:
- self.hostresults[host] = list(parentHash)
+ self.hostresults[host] = [parentHash]
else:
if parentHash in self.hostresults[host] or parentEvent.data == host:
self.sf.debug("Skipping host, " + host + ", already processed.")
return None
else:
- self.hostresults[host].append(parentHash)
+ self.hostresults[host] = self.hostresults[host] + [parentHash]
self.sf.debug("Found host: " + host)
# If the returned hostname is aliaseed to our
@@ -290,7 +288,7 @@ class sfp_dnsresolve(SpiderFootPlugin):
if htype.endswith("INTERNET_NAME"):
resolved = self.sf.resolveHost(host)
if htype == "INTERNET_NAME" and not resolved:
- evt = SpiderFootEvent("INTERNET_NAME_UNRESOLVED", host,
+ evt = SpiderFootEvent("INTERNET_NAME_UNRESOLVED", host,
self.__name__, parentEvent)
self.notifyListeners(evt)
return None
@@ -298,6 +296,7 @@ class sfp_dnsresolve(SpiderFootPlugin):
if not resolved:
return None
+ # Report the host
if host != parentEvent.data and htype != parentEvent.eventType:
evt = SpiderFootEvent(htype, host, self.__name__, parentEvent)
self.notifyListeners(evt)
@@ -311,11 +310,10 @@ class sfp_dnsresolve(SpiderFootPlugin):
# Try obtain the IPv6 address
ip6s = self.sf.resolveHost6(host)
- if not ip6s:
- return None
- for ip6 in ip6s:
- evt6 = SpiderFootEvent("IPV6_ADDRESS", ip6, self.__name__, evt)
- self.notifyListeners(evt6)
+ if ip6s:
+ for ip6 in ip6s:
+ evt6 = SpiderFootEvent("IPV6_ADDRESS", ip6, self.__name__, evt)
+ self.notifyListeners(evt6)
if htype == "AFFILIATE_INTERNET_NAME":
dom = self.sf.hostDomain(host, self.opts['_internettlds'])
diff --git a/modules/sfp_dnszonexfer.py b/modules/sfp_dnszonexfer.py
index ed0c7604..c5272162 100644
--- a/modules/sfp_dnszonexfer.py
+++ b/modules/sfp_dnszonexfer.py
@@ -26,11 +26,11 @@ class sfp_dnszonexfer(SpiderFootPlugin):
optdescs = {
}
- events = dict()
+ events = None
def setup(self, sfc, userOpts=dict()):
self.sf = sfc
- self.events = dict()
+ self.events = self.tempStorage()
self.__dataSource__ = "DNS"
diff --git a/modules/sfp_duckduckgo.py b/modules/sfp_duckduckgo.py
index f0d1f1ac..94b17352 100644
--- a/modules/sfp_duckduckgo.py
+++ b/modules/sfp_duckduckgo.py
@@ -16,9 +16,6 @@ from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
class sfp_duckduckgo(SpiderFootPlugin):
"""DuckDuckGo:Footprint,Investigate,Passive:Search Engines::Query DuckDuckGo's API for descriptive information about your target."""
-
-
-
# Default options
opts = {
"affiliatedomains": True
@@ -29,11 +26,11 @@ class sfp_duckduckgo(SpiderFootPlugin):
"affiliatedomains": "For affiliates, look up the domain name, not the hostname. This will usually return more meaningful information about the affiliate."
}
- results = list()
+ results = None
def setup(self, sfc, userOpts=dict()):
self.sf = sfc
- self.results = list()
+ self.results = self.tempStorage()
for opt in userOpts.keys():
self.opts[opt] = userOpts[opt]
@@ -64,7 +61,7 @@ class sfp_duckduckgo(SpiderFootPlugin):
self.sf.debug("Already did a search for " + eventData + ", skipping.")
return None
else:
- self.results.append(eventData)
+ self.results[eventData] = True
url = "https://api.duckduckgo.com/?q=" + eventData + "&format=json&pretty=1"
res = self.sf.fetchUrl(url, timeout=self.opts['_fetchtimeout'],
diff --git a/modules/sfp_email.py b/modules/sfp_email.py
index bdf34bf9..b63019f7 100644
--- a/modules/sfp_email.py
+++ b/modules/sfp_email.py
@@ -11,11 +11,7 @@
# Licence: GPL
# -------------------------------------------------------------------------------
-try:
- import re2 as re
-except ImportError:
- import re
-
+import re
from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
class sfp_email(SpiderFootPlugin):
diff --git a/modules/sfp_errors.py b/modules/sfp_errors.py
index 392e8eb1..28b3cb52 100644
--- a/modules/sfp_errors.py
+++ b/modules/sfp_errors.py
@@ -10,10 +10,7 @@
# Licence: GPL
# -------------------------------------------------------------------------------
-try:
- import re2 as re
-except ImportError:
- import re
+import re
from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
# Taken from Google Dorks on exploit-db.com
@@ -28,7 +25,6 @@ regexps = dict({
})
-
class sfp_errors(SpiderFootPlugin):
"""Errors:Footprint,Passive:Content Analysis::Identify common error messages in content like SQL errors, etc."""
@@ -74,7 +70,8 @@ class sfp_errors(SpiderFootPlugin):
if srcModuleName != "sfp_spider":
return None
- eventSource = event.sourceEvent.data
+ eventSource = event.actualSource
+
self.sf.debug("Received event, " + eventName + ", from " + srcModuleName)
if eventSource not in self.results.keys():
@@ -94,9 +91,9 @@ class sfp_errors(SpiderFootPlugin):
matches = re.findall(pat, eventData)
if len(matches) > 0 and regexpGrp not in self.results[eventSource]:
self.sf.info("Matched " + regexpGrp + " in content from " + eventSource)
- self.results[eventSource].append(regexpGrp)
+ self.results[eventSource] = self.results[eventSource] + [regexpGrp]
evt = SpiderFootEvent("ERROR_MESSAGE", regexpGrp,
- self.__name__, event.sourceEvent)
+ self.__name__, event)
self.notifyListeners(evt)
return None
diff --git a/modules/sfp_ethereum.py b/modules/sfp_ethereum.py
index 8b108e47..4df07666 100644
--- a/modules/sfp_ethereum.py
+++ b/modules/sfp_ethereum.py
@@ -11,10 +11,7 @@
# Licence: GPL
# -------------------------------------------------------------------------------
-try:
- import re2 as re
-except ImportError:
- import re
+import re
from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
class sfp_ethereum(SpiderFootPlugin):
diff --git a/modules/sfp_gravatar.py b/modules/sfp_gravatar.py
index 6bfea624..b66b5ec4 100644
--- a/modules/sfp_gravatar.py
+++ b/modules/sfp_gravatar.py
@@ -34,7 +34,7 @@ class sfp_gravatar(SpiderFootPlugin):
self.sf = sfc
self.__dataSource__ = 'Gravatar'
self.results = self.tempStorage()
- self.reportedUsers = dict()
+ self.reportedUsers = self.tempStorage()
for opt in userOpts.keys():
self.opts[opt] = userOpts[opt]
diff --git a/modules/sfp_hostsfilenet.py b/modules/sfp_hostsfilenet.py
index a3be9de7..0d686306 100644
--- a/modules/sfp_hostsfilenet.py
+++ b/modules/sfp_hostsfilenet.py
@@ -21,7 +21,6 @@ malchecks = {
}
}
-
class sfp_hostsfilenet(SpiderFootPlugin):
"""hosts-file.net Malicious Hosts:Investigate,Passive:Reputation Systems::Check if a host/domain is malicious according to hosts-file.net Malicious Hosts."""
@@ -44,11 +43,11 @@ class sfp_hostsfilenet(SpiderFootPlugin):
# Be sure to completely clear any class variables in setup()
# or you run the risk of data persisting between scan runs.
- results = list()
+ results = None
def setup(self, sfc, userOpts=dict()):
self.sf = sfc
- self.results = list()
+ self.results = self.tempStorage()
# Clear / reset any other class member variables here
# or you risk them persisting between threads.
@@ -89,6 +88,9 @@ class sfp_hostsfilenet(SpiderFootPlugin):
else:
self.sf.cachePut("sfmal_" + cid, data['content'])
+ if type(data['content']) != unicode:
+ data['content'] = unicode(data['content'], 'utf-8', errors='replace')
+
# Check for the domain and the hostname
if targetType == "domain" and "127.0.0.1\t" + targetDom + "\n" in data['content']:
self.sf.debug(targetDom + " found in " + check + " list.")
@@ -121,7 +123,7 @@ class sfp_hostsfilenet(SpiderFootPlugin):
self.sf.debug("Skipping " + eventData + ", already checked.")
return None
else:
- self.results.append(eventData)
+ self.results[eventData] = True
if eventName == 'CO_HOSTED_SITE' and not self.opts.get('checkcohosts', False):
return None
diff --git a/modules/sfp_junkfiles.py b/modules/sfp_junkfiles.py
index 68bfaa5c..ab592eb8 100644
--- a/modules/sfp_junkfiles.py
+++ b/modules/sfp_junkfiles.py
@@ -36,16 +36,16 @@ class sfp_junkfiles(SpiderFootPlugin):
}
results = None
- hosts = dict()
- skiphosts = dict()
- bases = dict()
+ hosts = None
+ skiphosts = None
+ bases = None
def setup(self, sfc, userOpts=dict()):
self.sf = sfc
self.results = self.tempStorage()
- self.hosts = dict()
- self.skiphosts = dict()
- self.bases = dict()
+ self.hosts = self.tempStorage()
+ self.skiphosts = self.tempStorage()
+ self.bases = self.tempStorage()
self.__dataSource__ = "Target Website"
for opt in userOpts.keys():
diff --git a/modules/sfp_names.py b/modules/sfp_names.py
index 5639c97d..df521786 100644
--- a/modules/sfp_names.py
+++ b/modules/sfp_names.py
@@ -68,7 +68,7 @@ class sfp_names(SpiderFootPlugin):
# If the source event is web content, check if the source URL was javascript
# or CSS, in which case optionally ignore it.
if eventName == "TARGET_WEB_CONTENT":
- url = event.sourceEvent.data
+ url = event.actualSource
if self.opts['filterjscss'] and (".js" in url or ".css" in url):
self.sf.debug("Ignoring web content from CSS/JS.")
return None
diff --git a/modules/sfp_neutrinoapi.py b/modules/sfp_neutrinoapi.py
index 1fe6ed16..ff44c300 100644
--- a/modules/sfp_neutrinoapi.py
+++ b/modules/sfp_neutrinoapi.py
@@ -56,7 +56,7 @@ class sfp_neutrinoapi(SpiderFootPlugin):
# https://www.neutrinoapi.com/api/phone-validate/
def queryPhoneValidate(self, qry):
res = self.sf.fetchUrl('https://neutrinoapi.com/phone-validate',
- postData="output-format=json&number=" + qry + "&user-id=" + self.opts['user_id'] + "&api-key=" + self.opts['api_key'],
+ postData={"output-format": "json", "number": qry, "user-id": self.opts['user_id'], "api-key": self.opts['api_key']},
timeout=self.opts['timeout'], useragent=self.opts['_useragent'])
return self.parseApiResponse(res)
@@ -65,7 +65,7 @@ class sfp_neutrinoapi(SpiderFootPlugin):
# https://www.neutrinoapi.com/api/ip-info/
def queryIpInfo(self, qry):
res = self.sf.fetchUrl("https://neutrinoapi.com/ip-info",
- postData="output-format=json&ip=" + qry + "&user-id=" + self.opts['user_id'] + "&api-key=" + self.opts['api_key'],
+ postData={"output-format": "json", "ip": qry, "user-id": self.opts['user_id'], "api-key": self.opts['api_key']},
timeout=self.opts['timeout'], useragent=self.opts['_useragent'])
return self.parseApiResponse(res)
@@ -74,7 +74,7 @@ class sfp_neutrinoapi(SpiderFootPlugin):
# https://www.neutrinoapi.com/api/ip-blocklist/
def queryIpBlocklist(self, qry):
res = self.sf.fetchUrl("https://neutrinoapi.com/ip-blocklist",
- postData="output-format=json&ip=" + qry + "&user-id=" + self.opts['user_id'] + "&api-key=" + self.opts['api_key'],
+ postData={"output-format": "json", "ip": qry, "user-id": self.opts['user_id'], "api-key": self.opts['api_key']},
timeout=self.opts['timeout'], useragent=self.opts['_useragent'])
return self.parseApiResponse(res)
@@ -83,7 +83,7 @@ class sfp_neutrinoapi(SpiderFootPlugin):
# https://www.neutrinoapi.com/api/host-reputation/
def queryHostReputation(self, qry):
res = self.sf.fetchUrl("https://neutrinoapi.com/host-reputation",
- postData="output-format=json&host=" + qry + "&user-id=" + self.opts['user_id'] + "&api-key=" + self.opts['api_key'],
+ postData={"output-format": "json", "host": qry, "user-id": self.opts['user_id'], "api-key": self.opts['api_key']},
timeout=self.opts['timeout'], useragent=self.opts['_useragent'])
return self.parseApiResponse(res)
diff --git a/modules/sfp_pageinfo.py b/modules/sfp_pageinfo.py
index b605bc8c..ae16a310 100644
--- a/modules/sfp_pageinfo.py
+++ b/modules/sfp_pageinfo.py
@@ -38,6 +38,7 @@ class sfp_pageinfo(SpiderFootPlugin):
def setup(self, sfc, userOpts=dict()):
self.sf = sfc
self.results = self.tempStorage()
+ self.__dataSource__ = "Target Website"
for opt in userOpts.keys():
self.opts[opt] = userOpts[opt]
@@ -65,7 +66,7 @@ class sfp_pageinfo(SpiderFootPlugin):
eventName = event.eventType
srcModuleName = event.module
eventData = event.data
- eventSource = event.sourceEvent.data # will be the URL of the raw data
+ eventSource = event.actualSource
self.sf.debug("Received event, " + eventName + ", from " + srcModuleName)
@@ -75,14 +76,14 @@ class sfp_pageinfo(SpiderFootPlugin):
self.sf.debug("Not gathering page info for external site " + eventSource)
return None
- if eventSource not in self.results.keys():
+ if eventSource not in self.results:
self.results[eventSource] = list()
else:
self.sf.debug("Already checked this page for a page type, skipping.")
return None
# Check the configured regexps to determine the page type
- for regexpGrp in regexps.keys():
+ for regexpGrp in regexps:
if regexpGrp in self.results[eventSource]:
continue
@@ -91,15 +92,14 @@ class sfp_pageinfo(SpiderFootPlugin):
matches = re.findall(rx, eventData)
if len(matches) > 0 and regexpGrp not in self.results[eventSource]:
self.sf.info("Matched " + regexpGrp + " in content from " + eventSource)
- self.results[eventSource].append(regexpGrp)
- evt = SpiderFootEvent(regexpGrp, eventSource,
- self.__name__, event.sourceEvent)
+ self.results[eventSource] = self.results[eventSource] + [regexpGrp]
+ evt = SpiderFootEvent(regexpGrp, eventSource, self.__name__, event)
self.notifyListeners(evt)
# If no regexps were matched, consider this a static page
if len(self.results[eventSource]) == 0:
self.sf.info("Treating " + eventSource + " as URL_STATIC")
- evt = SpiderFootEvent("URL_STATIC", eventSource, self.__name__, event.sourceEvent)
+ evt = SpiderFootEvent("URL_STATIC", eventSource, self.__name__, event)
self.notifyListeners(evt)
# Check for externally referenced Javascript pages
@@ -110,7 +110,7 @@ class sfp_pageinfo(SpiderFootPlugin):
if '://' in match and not self.getTarget().matches(self.sf.urlFQDN(match)):
self.sf.debug("Externally hosted Javascript found at: " + match)
evt = SpiderFootEvent("PROVIDER_JAVASCRIPT", match,
- self.__name__, event.sourceEvent)
+ self.__name__, event)
self.notifyListeners(evt)
return None
diff --git a/modules/sfp_phone.py b/modules/sfp_phone.py
index cac9c2a1..6e58f4c7 100644
--- a/modules/sfp_phone.py
+++ b/modules/sfp_phone.py
@@ -25,11 +25,11 @@ class sfp_phone(SpiderFootPlugin):
# Default options
opts = {}
- results = list()
+ results = None
def setup(self, sfc, userOpts=dict()):
self.sf = sfc
- self.results = list()
+ self.results = self.tempStorage()
for opt in userOpts.keys():
self.opts[opt] = userOpts[opt]
@@ -54,7 +54,7 @@ class sfp_phone(SpiderFootPlugin):
if sourceData in self.results:
return None
else:
- self.results.append(sourceData)
+ self.results[sourceData] = True
self.sf.debug("Received event, " + eventName + ", from " + srcModuleName)
diff --git a/modules/sfp_ripe.py b/modules/sfp_ripe.py
index 9839267c..8e0d8d3d 100644
--- a/modules/sfp_ripe.py
+++ b/modules/sfp_ripe.py
@@ -25,17 +25,17 @@ class sfp_ripe(SpiderFootPlugin):
results = None
currentEventSrc = None
- memCache = dict()
- nbreported = dict()
+ memCache = None
+ nbreported = None
keywords = None
lastContent = None
def setup(self, sfc, userOpts=dict()):
self.sf = sfc
self.results = self.tempStorage()
- self.memCache = dict()
+ self.memCache = self.tempStorage()
self.currentEventSrc = None
- self.nbreported = dict()
+ self.nbreported = self.tempStorage()
self.lastContent = None
for opt in userOpts.keys():
@@ -194,8 +194,9 @@ class sfp_ripe(SpiderFootPlugin):
# and the string supplied.
def findName(self, string):
# Simplest check to perform..
- if self.getTarget().getValue() in string:
- return True
+ for n in self.getTarget().getNames():
+ if n in string:
+ return True
if self.keywords is None:
self.keywords = self.sf.domainKeywords(self.getTarget().getNames(),
diff --git a/modules/sfp_s3bucket.py b/modules/sfp_s3bucket.py
index ee63b11b..79796a90 100644
--- a/modules/sfp_s3bucket.py
+++ b/modules/sfp_s3bucket.py
@@ -32,14 +32,14 @@ class sfp_s3bucket(SpiderFootPlugin):
"suffixes": "List of suffixes to append to domains tried as bucket names"
}
- results = list()
+ results = None
s3results = dict()
lock = None
def setup(self, sfc, userOpts=dict()):
self.sf = sfc
self.s3results = dict()
- self.results = list()
+ self.results = self.tempStorage()
self.lock = threading.Lock()
for opt in userOpts.keys():
@@ -56,12 +56,11 @@ class sfp_s3bucket(SpiderFootPlugin):
return ["CLOUD_STORAGE_BUCKET", "CLOUD_STORAGE_BUCKET_OPEN"]
def checkSite(self, url):
- res = self.sf.fetchUrl(url, timeout=10, useragent="SpiderFoot", noLog=True)
+ res = self.sf.fetchUrl(url, timeout=10, useragent="SpiderFoot")
if res['code'] not in [ "301", "302", "200" ] and \
(res['content'] is None or "NoSuchBucket" in res['content']):
- #self.sf.debug("Not a valid bucket: " + url)
- return None
+ self.sf.debug("Not a valid bucket: " + url)
else:
if "ListBucketResult" in res['content']:
with self.lock:
@@ -82,7 +81,7 @@ class sfp_s3bucket(SpiderFootPlugin):
return None
self.sf.info("Spawning thread to check bucket: " + site)
- t.append(threading.Thread(name='sfp_s3buckets_' + site,
+ t.append(threading.Thread(name='thread_sfp_s3buckets_' + site,
target=self.checkSite, args=(site,)))
t[i].start()
i += 1
@@ -91,7 +90,7 @@ class sfp_s3bucket(SpiderFootPlugin):
while running:
found = False
for rt in threading.enumerate():
- if rt.name.startswith("sfp_s3buckets_"):
+ if rt.name.startswith("thread_sfp_s3buckets_"):
found = True
if not found:
@@ -134,7 +133,7 @@ class sfp_s3bucket(SpiderFootPlugin):
if eventData in self.results:
return None
else:
- self.results.append(eventData)
+ self.results[eventData] = True
self.sf.debug("Received event, " + eventName + ", from " + srcModuleName)
@@ -165,7 +164,8 @@ class sfp_s3bucket(SpiderFootPlugin):
evt = SpiderFootEvent("CLOUD_STORAGE_BUCKET", bucket[0] + ":" + bucket[1], self.__name__, event)
self.notifyListeners(evt)
if bucket[2] != "0":
- evt = SpiderFootEvent("CLOUD_STORAGE_BUCKET_OPEN", bucket[0] + ": " + bucket[2] + " files found.",
+ bucketname = bucket[1].replace("//", "")
+ evt = SpiderFootEvent("CLOUD_STORAGE_BUCKET_OPEN", bucketname + ": " + bucket[2] + " files found.",
self.__name__, evt)
self.notifyListeners(evt)
diff --git a/modules/sfp_similar.py b/modules/sfp_similar.py
index b2a75333..4c037644 100644
--- a/modules/sfp_similar.py
+++ b/modules/sfp_similar.py
@@ -70,11 +70,11 @@ class sfp_similar(SpiderFootPlugin):
}
# Internal results tracking
- results = list()
+ results = None
def setup(self, sfc, userOpts=dict()):
self.sf = sfc
- self.results = list()
+ self.results = self.tempStorage()
self.__dataSource__ = "DNS"
for opt in userOpts.keys():
@@ -104,7 +104,7 @@ class sfp_similar(SpiderFootPlugin):
if dom in self.results:
return None
else:
- self.results.append(dom)
+ self.results[dom] = True
# Search for typos
pos = 0
diff --git a/modules/sfp_spider.py b/modules/sfp_spider.py
index 972e2d05..aab9418b 100644
--- a/modules/sfp_spider.py
+++ b/modules/sfp_spider.py
@@ -18,11 +18,10 @@ from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
class sfp_spider(SpiderFootPlugin):
"""Spider:Footprint,Investigate:Crawling and Scanning:slow:Spidering of web-pages to extract content for searching."""
-
# Default options
opts = {
'robotsonly': False, # only follow links specified by robots.txt
- 'pause': 1, # number of seconds to pause between fetches
+ 'pausesec': 0, # number of seconds to pause between fetches
'maxpages': 100, # max number of pages to fetch
'maxlevels': 3, # max number of levels to traverse within a site
'usecookies': True, # Use cookies?
@@ -42,8 +41,8 @@ class sfp_spider(SpiderFootPlugin):
# Option descriptions
optdescs = {
'robotsonly': "Only follow links specified by robots.txt?",
- 'pause': "Number of seconds to pause between fetches.",
'usecookies': "Accept and use cookies?",
+ 'pausesec': "Number of seconds to pause between page fetches.",
'start': "Prepend targets with these until you get a hit, to start spidering.",
'maxpages': "Maximum number of pages to fetch per starting point identified.",
'maxlevels': "Maximum levels to traverse per starting point (e.g. hostname or link identified by another module) identified.",
@@ -59,19 +58,19 @@ class sfp_spider(SpiderFootPlugin):
robotsRules = dict()
# Pages already fetched
- fetchedPages = dict()
+ fetchedPages = None
# Events for links identified
- urlEvents = dict()
+ urlEvents = None
# Tracked cookies per site
- siteCookies = dict()
+ siteCookies = None
def setup(self, sfc, userOpts=dict()):
self.sf = sfc
- self.fetchedPages = dict()
- self.urlEvents = dict()
- self.siteCookies = dict()
+ self.fetchedPages = self.tempStorage()
+ self.urlEvents = self.tempStorage()
+ self.siteCookies = self.tempStorage()
self.__dataSource__ = "Target Website"
for opt in userOpts.keys():
@@ -93,7 +92,8 @@ class sfp_spider(SpiderFootPlugin):
cookies = self.siteCookies[site]
# Fetch the contents of the supplied URL (object returned)
fetched = self.sf.fetchUrl(url, False, cookies,
- self.opts['_fetchtimeout'], self.opts['_useragent'])
+ self.opts['_fetchtimeout'], self.opts['_useragent'],
+ sizeLimit=10000000)
self.fetchedPages[url] = True
# Track cookies a site has sent, then send the back in subsquent requests
@@ -110,7 +110,7 @@ class sfp_spider(SpiderFootPlugin):
self.contentNotify(url, fetched, self.urlEvents[url])
if fetched['realurl'] is not None and fetched['realurl'] != url:
- self.sf.debug("Redirect of " + url + " to " + fetched['realurl'])
+ #self.sf.debug("Redirect of " + url + " to " + fetched['realurl'])
# Store the content for the redirect so that it isn't fetched again
self.fetchedPages[fetched['realurl']] = True
# Notify modules about the new link
@@ -144,7 +144,7 @@ class sfp_spider(SpiderFootPlugin):
def cleanLinks(self, links):
returnLinks = dict()
- for link in links.keys():
+ for link in links:
linkBase = self.sf.urlBaseUrl(link)
linkFQDN = self.sf.urlFQDN(link)
@@ -210,16 +210,17 @@ class sfp_spider(SpiderFootPlugin):
if sendcontent:
event = SpiderFootEvent("TARGET_WEB_CONTENT", httpresult['content'],
self.__name__, parentEvent)
+ event.actualSource = url
self.notifyListeners(event)
- if httpresult.get('headers') != None:
- event = SpiderFootEvent("WEBSERVER_HTTPHEADERS",
- json.dumps(httpresult['headers'], ensure_ascii=False),
- self.__name__, parentEvent)
- self.notifyListeners(event)
+ event = SpiderFootEvent("WEBSERVER_HTTPHEADERS", json.dumps(httpresult['headers'], ensure_ascii=False),
+ self.__name__, parentEvent)
+ event.actualSource = url
+ self.notifyListeners(event)
event = SpiderFootEvent("HTTP_CODE", str(httpresult['code']),
self.__name__, parentEvent)
+ event.actualSource = url
self.notifyListeners(event)
if not httpresult.get('headers'):
@@ -229,22 +230,25 @@ class sfp_spider(SpiderFootPlugin):
if ctype:
event = SpiderFootEvent("TARGET_WEB_CONTENT_TYPE", ctype,
self.__name__, parentEvent)
+ event.actualSource = url
self.notifyListeners(event)
-
# Trigger spidering off the following events..
# Spidering and search engines provide LINKED_URL_INTERNAL, and DNS lookups
# provide INTERNET_NAME.
def watchedEvents(self):
return ["LINKED_URL_INTERNAL", "INTERNET_NAME"]
+ # Don't notify me about events from myself
+ def watchOpts(self):
+ return [ 'noself' ]
+
# What events this module produces
# This is to support the end user in selecting modules based on events
# produced.
def producedEvents(self):
return ["WEBSERVER_HTTPHEADERS", "HTTP_CODE", "LINKED_URL_INTERNAL",
- "LINKED_URL_EXTERNAL", "TARGET_WEB_CONTENT",
- "TARGET_WEB_CONTENT_TYPE"]
+ "LINKED_URL_EXTERNAL", "TARGET_WEB_CONTENT", "TARGET_WEB_CONTENT_TYPE"]
# Some other modules may request we spider things
def handleEvent(self, event):
@@ -256,21 +260,19 @@ class sfp_spider(SpiderFootPlugin):
self.sf.debug("Received event, " + eventName + ", from " + srcModuleName)
if eventData in self.urlEvents:
- self.sf.debug("Ignoring " + eventName + " as already spidered or is being spidered.")
+ self.sf.debug("Ignoring " + eventData + " as already spidered or is being spidered.")
return None
+ else:
+ self.urlEvents[eventData] = event
# Don't spider links we find ourselves, obviously
if eventName == "LINKED_URL_INTERNAL" and "sfp_spider" in srcModuleName:
- self.sf.debug("Ignoring " + eventData + ", from self.")
return None
- self.urlEvents[eventData] = event
-
# Determine where to start spidering from if it's a INTERNET_NAME event
if eventName == "INTERNET_NAME":
for prefix in self.opts['start']:
- url = prefix + eventData
- res = self.sf.fetchUrl(url, timeout=self.opts['_fetchtimeout'],
+ res = self.sf.fetchUrl(prefix + eventData, timeout=self.opts['_fetchtimeout'],
useragent=self.opts['_useragent'])
if res['content'] is not None:
spiderTarget = prefix + eventData
@@ -325,7 +327,7 @@ class sfp_spider(SpiderFootPlugin):
links = dict()
# Fetch content from the new links
- for link in nextLinks.keys():
+ for link in nextLinks:
# Always skip links we've already fetched
if (link in self.fetchedPages):
self.sf.debug("Already fetched " + link + ", skipping.")
@@ -336,7 +338,7 @@ class sfp_spider(SpiderFootPlugin):
return None
self.sf.debug("Fetching fresh content from: " + link)
- time.sleep(self.opts['pause'])
+ time.sleep(self.opts['pausesec'])
freshLinks = self.processUrl(link)
if freshLinks is not None:
links.update(freshLinks)
diff --git a/modules/sfp_sslcert.py b/modules/sfp_sslcert.py
index cf411dc0..ee8233da 100644
--- a/modules/sfp_sslcert.py
+++ b/modules/sfp_sslcert.py
@@ -80,8 +80,8 @@ class sfp_sslcert(SpiderFootPlugin):
# Handle URLs containing port numbers
u = urlparse.urlparse(eventData)
port = 443
- if url.port:
- port = url.port
+ if u.port:
+ port = u.port
fqdn = self.sf.urlFQDN(eventData.lower())
else:
fqdn = eventData
diff --git a/modules/sfp_strangeheaders.py b/modules/sfp_strangeheaders.py
index 0c2ed2fd..99b4083c 100644
--- a/modules/sfp_strangeheaders.py
+++ b/modules/sfp_strangeheaders.py
@@ -24,7 +24,6 @@ headers = ["access-control-allow-origin", "accept-ranges", "age", "allow", "cach
"x-frame-options", "x-xss-protection", "content-security-policy", "x-content-security-policy",
"x-webkit-csp", "x-content-type-options", "x-powered-by", "x-ua-compatible"]
-
class sfp_strangeheaders(SpiderFootPlugin):
"""Strange Headers:Footprint,Passive:Content Analysis::Obtain non-standard HTTP headers returned by web servers."""
@@ -56,9 +55,7 @@ class sfp_strangeheaders(SpiderFootPlugin):
eventName = event.eventType
srcModuleName = event.module
eventData = event.data
- parentEvent = event.sourceEvent
- eventSource = event.sourceEvent.data
-
+ eventSource = event.actualSource
self.sf.debug("Received event, " + eventName + ", from " + srcModuleName)
if eventSource in self.results:
@@ -82,7 +79,7 @@ class sfp_strangeheaders(SpiderFootPlugin):
if key.lower() not in headers:
val = key + ": " + jdata[key]
evt = SpiderFootEvent("WEBSERVER_STRANGEHEADER", val,
- self.__name__, parentEvent)
+ self.__name__, event)
self.notifyListeners(evt)
# End of sfp_strangeheaders class
diff --git a/modules/sfp_tldsearch.py b/modules/sfp_tldsearch.py
index 987b5123..a176127c 100644
--- a/modules/sfp_tldsearch.py
+++ b/modules/sfp_tldsearch.py
@@ -14,28 +14,27 @@
import random
import threading
import time
+import dns
from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
-
class sfp_tldsearch(SpiderFootPlugin):
"""TLD Search:Footprint:DNS:slow:Search all Internet TLDs for domains with the same name as the target (this can be very slow.)"""
# Default options
opts = {
- 'activeonly': True, # Only report domains that have content (try to fetch the page)
+ 'activeonly': False, # Only report domains that have content (try to fetch the page)
'skipwildcards': True,
- 'maxthreads': 100
+ '_maxthreads': 50
}
# Option descriptions
optdescs = {
'activeonly': "Only report domains that have content (try to fetch the page)?",
- "skipwildcards": "Skip TLDs and sub-TLDs that have wildcard DNS.",
- "maxthreads": "Number of simultaneous DNS resolutions to perform at once."
+ "skipwildcards": "Skip TLDs and sub-TLDs that have wildcard DNS."
}
# Internal results tracking
- results = list()
+ results = None
# Track TLD search results between threads
tldResults = dict()
@@ -43,7 +42,7 @@ class sfp_tldsearch(SpiderFootPlugin):
def setup(self, sfc, userOpts=dict()):
self.sf = sfc
- self.results = list()
+ self.results = self.tempStorage()
self.__dataSource__ = "DNS"
self.lock = threading.Lock()
@@ -61,16 +60,25 @@ class sfp_tldsearch(SpiderFootPlugin):
return ["SIMILARDOMAIN"]
def tryTld(self, target, tld):
+ resolver = dns.resolver.Resolver()
+ resolver.timeout = 1
+ resolver.lifetime = 1
+ resolver.search = list()
+
if self.opts['skipwildcards'] and self.sf.checkDnsWildcard(tld):
return None
- addrs = self.sf.resolveHost(target)
- if not addrs:
+ try:
+ addrs = self.sf.resolveHost(target, resolver)
+ if not addrs:
+ with self.lock:
+ self.tldResults[target] = False
+ else:
+ with self.lock:
+ self.tldResults[target] = True
+ except BaseException as e:
with self.lock:
self.tldResults[target] = False
- else:
- with self.lock:
- self.tldResults[target] = True
def tryTldWrapper(self, tldList, sourceEvent):
self.tldResults = dict()
@@ -82,7 +90,7 @@ class sfp_tldsearch(SpiderFootPlugin):
self.sf.info("Spawning threads to check TLDs: " + str(tldList))
for pair in tldList:
(domain, tld) = pair
- tn = 'sfp_tldsearch_' + str(random.SystemRandom().randint(0, 999999999))
+ tn = 'thread_sfp_tldsearch_' + str(random.SystemRandom().randint(0, 999999999))
t.append(threading.Thread(name=tn, target=self.tryTld, args=(domain, tld,)))
t[i].start()
i += 1
@@ -91,22 +99,22 @@ class sfp_tldsearch(SpiderFootPlugin):
while running:
found = False
for rt in threading.enumerate():
- if rt.name.startswith("sfp_tldsearch_"):
+ if rt.name.startswith("thread_sfp_tldsearch_"):
found = True
if not found:
running = False
- time.sleep(1)
+ time.sleep(0.1)
- for res in self.tldResults.keys():
+ for res in self.tldResults:
if self.tldResults[res] and res not in self.results:
self.sendEvent(sourceEvent, res)
# Store the result internally and notify listening modules
def sendEvent(self, source, result):
self.sf.info("Found a TLD with the target's name: " + result)
- self.results.append(result)
+ self.results[result] = True
# Inform listening modules
if self.opts['activeonly']:
@@ -131,7 +139,7 @@ class sfp_tldsearch(SpiderFootPlugin):
if eventData in self.results:
return None
else:
- self.results.append(eventData)
+ self.results[eventData] = True
keyword = self.sf.domainKeyword(eventData, self.opts['_internettlds'])
self.sf.debug("Keyword extracted from " + eventData + ": " + keyword)
@@ -140,7 +148,7 @@ class sfp_tldsearch(SpiderFootPlugin):
if keyword in self.results:
return None
else:
- self.results.append(keyword)
+ self.results[keyword] = True
# Look through all TLDs for the existence of this target keyword
for tld in self.opts['_internettlds']:
@@ -163,7 +171,7 @@ class sfp_tldsearch(SpiderFootPlugin):
if self.checkForStop():
return None
- if len(targetList) <= self.opts['maxthreads']:
+ if len(targetList) <= self.opts['_maxthreads']:
targetList.append([tryDomain, tld])
else:
self.tryTldWrapper(targetList, event)
diff --git a/modules/sfp_webanalytics.py b/modules/sfp_webanalytics.py
index ae8aaf62..8cf25bd5 100644
--- a/modules/sfp_webanalytics.py
+++ b/modules/sfp_webanalytics.py
@@ -43,7 +43,6 @@ class sfp_webanalytics(SpiderFootPlugin):
srcModuleName = event.module
eventData = event.data
sourceData = self.sf.hashstring(eventData)
- parentEvent = event.sourceEvent
if sourceData in self.results:
self.sf.debug("Skipping " + eventData + " as already mapped.")
@@ -67,7 +66,7 @@ class sfp_webanalytics(SpiderFootPlugin):
self.sf.debug("Google Analytics match: " + m)
evt = SpiderFootEvent("WEB_ANALYTICS_ID",
"Google Analytics: " + m,
- self.__name__, parentEvent)
+ self.__name__, event)
self.notifyListeners(evt)
# Google AdSense
@@ -79,7 +78,7 @@ class sfp_webanalytics(SpiderFootPlugin):
self.sf.debug("Google AdSense match: " + m)
evt = SpiderFootEvent("WEB_ANALYTICS_ID",
"Google AdSense: " + m,
- self.__name__, parentEvent)
+ self.__name__, event)
self.notifyListeners(evt)
# Google Website Verification
@@ -89,7 +88,7 @@ class sfp_webanalytics(SpiderFootPlugin):
self.sf.debug("Google Site Verification match: " + m)
evt = SpiderFootEvent("WEB_ANALYTICS_ID",
"Google Site Verification: " + m,
- self.__name__, parentEvent)
+ self.__name__, event)
self.notifyListeners(evt)
matches = re.findall(r'<meta name="verify-v1" content="([a-z0-9\-\+_=]{43,44})"', eventData, re.IGNORECASE)
@@ -97,7 +96,7 @@ class sfp_webanalytics(SpiderFootPlugin):
self.sf.debug("Google Site Verification match: " + m)
evt = SpiderFootEvent("WEB_ANALYTICS_ID",
"Google Site Verification: " + m,
- self.__name__, parentEvent)
+ self.__name__, event)
self.notifyListeners(evt)
# Quantcast
@@ -107,7 +106,7 @@ class sfp_webanalytics(SpiderFootPlugin):
self.sf.debug("Quantcast match: " + m)
evt = SpiderFootEvent("WEB_ANALYTICS_ID",
"Quantcast: " + m,
- self.__name__, parentEvent)
+ self.__name__, event)
self.notifyListeners(evt)
# Ahrefs Site Verification
@@ -116,7 +115,7 @@ class sfp_webanalytics(SpiderFootPlugin):
self.sf.debug("Ahrefs Site Verification match: " + m)
evt = SpiderFootEvent("WEB_ANALYTICS_ID",
"Ahrefs Site Verification: " + m,
- self.__name__, parentEvent)
+ self.__name__, event)
self.notifyListeners(evt)
if eventName == 'DNS_TEXT':
@@ -126,7 +125,7 @@ class sfp_webanalytics(SpiderFootPlugin):
for m in matches:
evt = SpiderFootEvent("WEB_ANALYTICS_ID",
"Google Site Verification: " + m,
- self.__name__, parentEvent)
+ self.__name__, event)
self.notifyListeners(evt)
# LogMeIn Domain Verification
@@ -135,14 +134,14 @@ class sfp_webanalytics(SpiderFootPlugin):
for m in matches:
evt = SpiderFootEvent("WEB_ANALYTICS_ID",
"LogMeIn Domain Verification: " + m,
- self.__name__, parentEvent)
+ self.__name__, event)
self.notifyListeners(evt)
matches = re.findall(r'logmein-verification-code=([a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12})$', eventData.strip(), re.IGNORECASE)
for m in matches:
evt = SpiderFootEvent("WEB_ANALYTICS_ID",
"LogMeIn Domain Verification: " + m,
- self.__name__, parentEvent)
+ self.__name__, event)
self.notifyListeners(evt)
# DocuSign Domain Verification
@@ -151,7 +150,7 @@ class sfp_webanalytics(SpiderFootPlugin):
for m in matches:
evt = SpiderFootEvent("WEB_ANALYTICS_ID",
"DocuSign Domain Verification: " + m,
- self.__name__, parentEvent)
+ self.__name__, event)
self.notifyListeners(evt)
# GlobalSign Site Verification
@@ -160,7 +159,7 @@ class sfp_webanalytics(SpiderFootPlugin):
for m in matches:
evt = SpiderFootEvent("WEB_ANALYTICS_ID",
"GlobalSign Site Verification: " + m,
- self.__name__, parentEvent)
+ self.__name__, event)
self.notifyListeners(evt)
# Atlassian Domain Verification
@@ -169,7 +168,7 @@ class sfp_webanalytics(SpiderFootPlugin):
for m in matches:
evt = SpiderFootEvent("WEB_ANALYTICS_ID",
"Atlassian Domain Verification: " + m,
- self.__name__, parentEvent)
+ self.__name__, event)
self.notifyListeners(evt)
# Adobe IDP Site Verification
@@ -178,14 +177,14 @@ class sfp_webanalytics(SpiderFootPlugin):
for m in matches:
evt = SpiderFootEvent("WEB_ANALYTICS_ID",
"Adobe IDP Site Verification: " + m,
- self.__name__, parentEvent)
+ self.__name__, event)
self.notifyListeners(evt)
matches = re.findall(r'adobe-idp-site-verification=([a-f0-9]{64})$', eventData.strip(), re.IGNORECASE)
for m in matches:
evt = SpiderFootEvent("WEB_ANALYTICS_ID",
"Adobe IDP Site Verification: " + m,
- self.__name__, parentEvent)
+ self.__name__, event)
self.notifyListeners(evt)
# Adobe Domain Verification
@@ -194,7 +193,7 @@ class sfp_webanalytics(SpiderFootPlugin):
for m in matches:
evt = SpiderFootEvent("WEB_ANALYTICS_ID",
"Adobe Domain Verification: " + m,
- self.__name__, parentEvent)
+ self.__name__, event)
self.notifyListeners(evt)
# Stripe Domain Verification
@@ -203,7 +202,7 @@ class sfp_webanalytics(SpiderFootPlugin):
for m in matches:
evt = SpiderFootEvent("WEB_ANALYTICS_ID",
"Stripe Domain Verification: " + m,
- self.__name__, parentEvent)
+ self.__name__, event)
self.notifyListeners(evt)
@@ -213,7 +212,7 @@ class sfp_webanalytics(SpiderFootPlugin):
for m in matches:
evt = SpiderFootEvent("WEB_ANALYTICS_ID",
"TeamViewer SSO Verification: " + m,
- self.__name__, parentEvent)
+ self.__name__, event)
self.notifyListeners(evt)
# Aliyun Site Verification
@@ -221,7 +220,7 @@ class sfp_webanalytics(SpiderFootPlugin):
for m in matches:
evt = SpiderFootEvent("WEB_ANALYTICS_ID",
"Aliyun Site Verification: " + m,
- self.__name__, parentEvent)
+ self.__name__, event)
self.notifyListeners(evt)
# Facebook Domain Verification
@@ -230,7 +229,7 @@ class sfp_webanalytics(SpiderFootPlugin):
for m in matches:
evt = SpiderFootEvent("WEB_ANALYTICS_ID",
"Facebook Domain Verification: " + m,
- self.__name__, parentEvent)
+ self.__name__, event)
self.notifyListeners(evt)
# Citrix Domain Verification
@@ -238,7 +237,7 @@ class sfp_webanalytics(SpiderFootPlugin):
for m in matches:
evt = SpiderFootEvent("WEB_ANALYTICS_ID",
"Citrix Domain Verification: " + m,
- self.__name__, parentEvent)
+ self.__name__, event)
self.notifyListeners(evt)
# Dropbox Domain Verification
@@ -247,7 +246,7 @@ class sfp_webanalytics(SpiderFootPlugin):
for m in matches:
evt = SpiderFootEvent("WEB_ANALYTICS_ID",
"Dropbox Domain Verification: " + m,
- self.__name__, parentEvent)
+ self.__name__, event)
self.notifyListeners(evt)
# Detectify Domain Verification
@@ -256,7 +255,7 @@ class sfp_webanalytics(SpiderFootPlugin):
for m in matches:
evt = SpiderFootEvent("WEB_ANALYTICS_ID",
"Detectify Domain Verification: " + m,
- self.__name__, parentEvent)
+ self.__name__, event)
self.notifyListeners(evt)
# Drift Domain Verification
@@ -264,7 +263,7 @@ class sfp_webanalytics(SpiderFootPlugin):
for m in matches:
evt = SpiderFootEvent("WEB_ANALYTICS_ID",
"Drift Domain Verification: " + m,
- self.__name__, parentEvent)
+ self.__name__, event)
self.notifyListeners(evt)
# Ahrefs Site Verification
@@ -273,7 +272,7 @@ class sfp_webanalytics(SpiderFootPlugin):
for m in matches:
evt = SpiderFootEvent("WEB_ANALYTICS_ID",
"Ahrefs Site Verification: " + m,
- self.__name__, parentEvent)
+ self.__name__, event)
self.notifyListeners(evt)
# Statuspage.io Domain Verification
@@ -282,7 +281,7 @@ class sfp_webanalytics(SpiderFootPlugin):
for m in matches:
evt = SpiderFootEvent("WEB_ANALYTICS_ID",
"Statuspage Domain Verification: " + m,
- self.__name__, parentEvent)
+ self.__name__, event)
self.notifyListeners(evt)
# Zoom.us Domain Verification
@@ -291,7 +290,7 @@ class sfp_webanalytics(SpiderFootPlugin):
for m in matches:
evt = SpiderFootEvent("WEB_ANALYTICS_ID",
"Zoom.us Domain Verification: " + m,
- self.__name__, parentEvent)
+ self.__name__, event)
self.notifyListeners(evt)
# Mail.ru Domain Verification
@@ -299,7 +298,7 @@ class sfp_webanalytics(SpiderFootPlugin):
for m in matches:
evt = SpiderFootEvent("WEB_ANALYTICS_ID",
"Mail.ru Domain Verification: " + m,
- self.__name__, parentEvent)
+ self.__name__, event)
self.notifyListeners(evt)
# Yandex Domain Verification
@@ -307,7 +306,7 @@ class sfp_webanalytics(SpiderFootPlugin):
for m in matches:
evt = SpiderFootEvent("WEB_ANALYTICS_ID",
"Yandex Domain Verification: " + m,
- self.__name__, parentEvent)
+ self.__name__, event)
self.notifyListeners(evt)
# Brave Ledger Verification
diff --git a/modules/sfp_webframework.py b/modules/sfp_webframework.py
index 97f68d0d..612c8c88 100644
--- a/modules/sfp_webframework.py
+++ b/modules/sfp_webframework.py
@@ -25,7 +25,6 @@ regexps = dict({
"Wordpress": list(['\/wp-includes\/', '\/wp-content\/'])
})
-
class sfp_webframework(SpiderFootPlugin):
"""Web Framework:Footprint,Passive:Content Analysis::Identify the usage of popular web frameworks like jQuery, YUI and others."""
@@ -66,18 +65,15 @@ class sfp_webframework(SpiderFootPlugin):
eventName = event.eventType
srcModuleName = event.module
eventData = event.data
+ eventSource = event.actualSource
# We only want web content
if srcModuleName != "sfp_spider":
return None
- # If you are processing TARGET_WEB_CONTENT, this is how you would get the
- # source of that raw data (e.g. a URL.)
- eventSource = event.sourceEvent.data
-
self.sf.debug("Received event, " + eventName + ", from " + srcModuleName)
- if eventSource not in self.results.keys():
+ if eventSource not in self.results:
self.results[eventSource] = list()
# We only want web content for pages on the target site
@@ -94,16 +90,11 @@ class sfp_webframework(SpiderFootPlugin):
matches = re.findall(pat, eventData)
if len(matches) > 0 and regexpGrp not in self.results[eventSource]:
self.sf.info("Matched " + regexpGrp + " in content from " + eventSource)
- self.results[eventSource].append(regexpGrp)
+ self.results[eventSource] = self.results[eventSource] + [regexpGrp]
evt = SpiderFootEvent("URL_WEB_FRAMEWORK", regexpGrp,
- self.__name__, event.sourceEvent)
+ self.__name__, event)
self.notifyListeners(evt)
return None
- # If you intend for this module to act on its own (e.g. not solely rely
- # on events from other modules, then you need to have a start() method
- # and within that method call self.checkForStop() to see if you've been
- # politely asked by the controller to stop your activities (user abort.)
-
# End of sfp_webframework class
diff --git a/modules/sfp_webserver.py b/modules/sfp_webserver.py
index 851927ab..fd0f31f0 100644
--- a/modules/sfp_webserver.py
+++ b/modules/sfp_webserver.py
@@ -14,7 +14,6 @@
import json
from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
-
class sfp_webserver(SpiderFootPlugin):
"""Web Server:Footprint,Investigate,Passive:Content Analysis::Obtain web server banners to identify versions of web servers being used."""
@@ -47,8 +46,7 @@ class sfp_webserver(SpiderFootPlugin):
eventName = event.eventType
srcModuleName = event.module
eventData = event.data
- parentEvent = event.sourceEvent
- eventSource = event.sourceEvent.data
+ eventSource = event.actualSource
self.sf.debug("Received event, " + eventName + ", from " + srcModuleName)
if eventSource in self.results:
@@ -72,10 +70,10 @@ class sfp_webserver(SpiderFootPlugin):
if 'location' in jdata:
if jdata['location'].startswith('http://') or jdata['location'].startswith('https://'):
if self.getTarget().matches(self.sf.urlFQDN(jdata['location'])):
- evt = SpiderFootEvent('LINKED_URL_INTERNAL', jdata['location'], self.__name__, parentEvent)
+ evt = SpiderFootEvent('LINKED_URL_INTERNAL', jdata['location'], self.__name__, event)
self.notifyListeners(evt)
else:
- evt = SpiderFootEvent('LINKED_URL_EXTERNAL', jdata['location'], self.__name__, parentEvent)
+ evt = SpiderFootEvent('LINKED_URL_EXTERNAL', jdata['location'], self.__name__, event)
self.notifyListeners(evt)
# Check CSP header for linked URLs
@@ -84,10 +82,10 @@ class sfp_webserver(SpiderFootPlugin):
for string in directive.split(' '):
if string.startswith('http://') or string.startswith('https://'):
if self.getTarget().matches(self.sf.urlFQDN(string)):
- evt = SpiderFootEvent('LINKED_URL_INTERNAL', string, self.__name__, parentEvent)
+ evt = SpiderFootEvent('LINKED_URL_INTERNAL', string, self.__name__, event)
self.notifyListeners(evt)
else:
- evt = SpiderFootEvent('LINKED_URL_EXTERNAL', string, self.__name__, parentEvent)
+ evt = SpiderFootEvent('LINKED_URL_EXTERNAL', string, self.__name__, event)
self.notifyListeners(evt)
# Could apply some smarts here, for instance looking for certain
@@ -96,14 +94,14 @@ class sfp_webserver(SpiderFootPlugin):
# and other errors to see what the header looks like.
if 'server' in jdata:
evt = SpiderFootEvent("WEBSERVER_BANNER", jdata['server'],
- self.__name__, parentEvent)
+ self.__name__, event)
self.notifyListeners(evt)
self.sf.info("Found web server: " + jdata['server'] + " (" + eventSource + ")")
if 'x-powered-by' in jdata:
evt = SpiderFootEvent("WEBSERVER_TECHNOLOGY", jdata['x-powered-by'],
- self.__name__, parentEvent)
+ self.__name__, event)
self.notifyListeners(evt)
return None
@@ -127,7 +125,7 @@ class sfp_webserver(SpiderFootPlugin):
tech = "PHP"
if tech is not None:
- evt = SpiderFootEvent("WEBSERVER_TECHNOLOGY", tech, self.__name__, parentEvent)
+ evt = SpiderFootEvent("WEBSERVER_TECHNOLOGY", tech, self.__name__, event)
self.notifyListeners(evt)
# End of sfp_webserver class
diff --git a/sflib.py b/sflib.py
index f691b374..4766a332 100644
--- a/sflib.py
+++ b/sflib.py
@@ -898,7 +898,7 @@ class SpiderFoot:
#
# Return a normalised resolution or None if not resolved.
- def resolveHost(self, host):
+ def resolveHost(self, host, resolver=None):
try:
# IDNA-encode the hostname in case it contains unicode
if type(host) != unicode:
@@ -906,9 +906,12 @@ class SpiderFoot:
else:
host = host.encode("idna")
- addrs = self.normalizeDNS(socket.gethostbyname_ex(host))
+ if not resolver:
+ addrs = self.normalizeDNS(socket.gethostbyname_ex(host))
+ else:
+ addrs = self.normalizeDNS(resolver.query(host))
if len(addrs) > 0:
- return addrs
+ return set(addrs)
return None
except BaseException as e:
self.debug("Unable to resolve " + host + ": " + str(e))
@@ -921,7 +924,7 @@ class SpiderFoot:
try:
addrs = self.normalizeDNS(socket.gethostbyaddr(ipaddr))
if len(addrs) > 0:
- return addrs
+ return set(addrs)
return None
except BaseException as e:
self.debug("Unable to resolve " + ipaddr + " (" + str(e) + ")")
@@ -938,7 +941,7 @@ class SpiderFoot:
if len(addrs) < 1:
return None
self.debug("Resolved " + hostname + " to IPv6: " + str(addrs))
- return addrs
+ return set(addrs)
except BaseException as e:
self.debug("Unable to IPv6 resolve " + hostname + " (" + str(e) + ")")
return None
@@ -956,6 +959,43 @@ class SpiderFoot:
return False
+ # Resolve alternative names for a given target
+ def resolveTargets(self, target, validateReverse):
+ ret = list()
+ t = target.getType()
+ v = target.getValue()
+
+ if t == "IP_ADDRESS":
+ r = self.resolveIP(v)
+ if r:
+ ret.extend(r)
+ if t == "INTERNET_NAME":
+ r = self.resolveHost(v)
+ if r:
+ ret.extend(r)
+ if t == "NETBLOCK_OWNER":
+ for addr in IPNetwork(v):
+ ipaddr = str(addr)
+ if ipaddr.split(".")[3] in ['255', '0']:
+ continue
+ if '255' in ipaddr.split("."):
+ continue
+ ret.append(ipaddr)
+
+ # Add the reverse-resolved hostnames as aliases too..
+ names = self.resolveIP(ipaddr)
+ if names:
+ if validateReverse:
+ for host in names:
+ chk = self.resolveHost(host)
+ if ipaddr in chk:
+ ret.append(host)
+ else:
+ ret.extend(names)
+ if len(ret) > 0:
+ return set(ret)
+ return None
+
# Create a safe socket that's using SOCKS/TOR if it was enabled
def safeSocket(self, host, port, timeout):
sock = socket.create_connection((host, int(port)), int(timeout))
@@ -1300,15 +1340,43 @@ class SpiderFoot:
if url is None:
return None
- # Clean the URL
- if type(url) != unicode:
- url = unicode(url, 'utf-8', errors='replace')
+ proxies = dict()
+ if self.opts['_socks1type']:
+ neverProxyNames = [ self.opts['_socks2addr'] ]
+ neverProxySubnets = [ "192.168.", "127.", "10." ]
+ host = self.urlFQDN(url)
- # Convert any unicode chars in the URL
- url = str(self.urlEncodeUnicode(url))
+ # Completely on or off?
+ if self.opts['_socks1type']:
+ proxy = True
+ else:
+ proxy = False
+
+ # Never proxy these system/internal locations
+ # This logic also exists in ext/socks.py and may not be
+ # needed anymore.
+ if host in neverProxyNames:
+ proxy = False
+ for s in neverProxyNames:
+ if host.endswith(s):
+ proxy = False
+ for sub in neverProxySubnets:
+ if host.startswith(sub):
+ proxy = False
+
+ if proxy:
+ self.debug("Using proxy for " + host)
+ self.debug("Proxy set to " + self.opts['_socks2addr'] + ":" + str(self.opts['_socks3port']))
+ proxies = {
+ 'http': 'socks5h://' + self.opts['_socks2addr'] + ":" + str(self.opts['_socks3port']),
+ 'https': 'socks5h://' + self.opts['_socks2addr'] + ":" + str(self.opts['_socks3port'])
+ }
+ else:
+ self.debug("Not using proxy for " + host)
try:
header = dict()
+ btime = time.time()
if type(useragent) is list:
header['User-Agent'] = random.SystemRandom().choice(useragent)
else:
@@ -1317,15 +1385,19 @@ class SpiderFoot:
# Add custom headers
if headers is not None:
for k in headers.keys():
- header[k] = headers[k]
+ if type(headers[k]) != unicode:
+ header[k] = unicode(headers[k], 'utf-8', errors='replace')
+ else:
+ header[k] = headers[k]
if sizeLimit or headOnly:
if not noLog:
self.info("Fetching (HEAD only): " + url + \
- " [timeout: " + \
+ " [user-agent: " + header['User-Agent'] + "] [timeout: " + \
str(timeout) + "]")
- hdr = requests.head(url, headers=header, verify=verify, timeout=timeout)
+ hdr = requests.head(url, headers=header, proxies=proxies,
+ verify=False, timeout=timeout)
size = int(hdr.headers.get('content-length', 0))
result['realurl'] = hdr.headers.get('location', url)
result['code'] = str(hdr.status_code)
@@ -1339,74 +1411,99 @@ class SpiderFoot:
if result['realurl'] != url:
if not noLog:
self.info("Fetching (HEAD only): " + url + \
- " [timeout: " + \
+ " [user-agent: " + header['User-Agent'] + "] [timeout: " + \
str(timeout) + "]")
- hdr = requests.head(result['realurl'], headers=header, verify=verify)
+ hdr = requests.head(result['realurl'], headers=header, proxies=proxies,
+ verify=False, timeout=timeout)
size = int(hdr.headers.get('content-length', 0))
result['realurl'] = hdr.headers.get('location', result['realurl'])
result['code'] = str(hdr.status_code)
if size > sizeLimit:
return result
-
- req = urllib2.Request(url, postData, header)
if cookies is not None:
- req.add_header('cookie', cookies)
+ #req.add_header('cookie', cookies)
if not noLog:
self.info("Fetching (incl. cookies): " + url + \
- " [timeout: " + \
+ " [user-agent: " + header['User-Agent'] + "] [timeout: " + \
str(timeout) + "]")
else:
if not noLog:
- self.info("Fetching: " + url + " [timeout: " + str(timeout) + "]")
-
+ self.info("Fetching: " + url + " [user-agent: " + \
+ header['User-Agent'] + "] [timeout: " + str(timeout) + "]")
+
+ #
+ # MAKE THE REQUEST
+ #
+ if postData:
+ res = requests.post(url, data=postData, headers=header, proxies=proxies,
+ cookies=cookies, timeout=timeout, verify=False)
+ else:
+ res = requests.get(url, headers=header, proxies=proxies,
+ cookies=cookies, timeout=timeout, verify=False)
result['headers'] = dict()
- opener = urllib2.build_opener(SmartRedirectHandler())
- fullPage = opener.open(req, timeout=timeout)
- content = fullPage.read()
-
- for k, v in fullPage.info().items():
- result['headers'][k.lower()] = v
+ for h in res.headers:
+ if type(h) != unicode:
+ hu = unicode(h, 'utf-8', errors='replace')
+ else:
+ hu = h
+ v = res.headers.get(h)
+ if type(v) != unicode:
+ vu = unicode(v, 'utf-8', errors='replace')
+ else:
+ vu = v
+ result['headers'][hu.lower()] = vu
- # Content is compressed
- if 'gzip' in result['headers'].get('content-encoding', ''):
- content = gzip.GzipFile(fileobj=StringIO.StringIO(content)).read()
+ # Sometimes content exceeds the size limit after decompression
+ if sizeLimit and len(res.content) > sizeLimit:
+ self.debug("Content exceeded size limit, so returning no data just headers")
+ result['realurl'] = res.url
+ result['code'] = str(res.status_code)
+ return result
+ if 'refresh' in result['headers']:
+ try:
+ newurl = result['headers']['refresh'].split(";url=")[1]
+ except BaseException as e:
+ self.debug("Refresh header found but was not parsable: " + result['headers']['refresh'])
+ return result
+ self.debug("Refresh header found, re-directing to " + newurl)
+ return self.fetchUrl(newurl, fatal, cookies, timeout,
+ useragent, headers, noLog, postData,
+ dontMangle, sizeLimit, headOnly)
+
+ #print "FOR: " + url
+ #print "HEADERS: " + str(result['headers'])
+ result['realurl'] = res.url
+ result['code'] = str(res.status_code)
if dontMangle:
- result['content'] = content
+ result['content'] = res.content
else:
- result['content'] = unicode(content, 'utf-8', errors='replace')
-
- #print("FOR: " + url)
- #print("HEADERS: " + str(result['headers']))
- result['realurl'] = fullPage.geturl()
- result['code'] = str(fullPage.getcode())
- result['status'] = 'OK'
- except urllib2.HTTPError as h:
- if not noLog:
- self.info("HTTP code " + str(h.code) + " encountered for " + url)
- # Capture the HTTP error code
- result['code'] = str(h.code)
- for k, v in h.info().items():
- result['headers'][k.lower()] = v
+ result['content'] = unicode(res.content, 'utf-8', errors='replace')
if fatal:
- self.fatal('URL could not be fetched (' + str(h.code) + ')')
- except urllib2.URLError as e:
- if not noLog:
- self.info("Error fetching " + url + "(" + str(e) + ")")
- result['status'] = str(e)
- if fatal:
- self.fatal('URL could not be fetched (' + str(e) + ')')
+ res.raise_for_status()
+ except requests.exceptions.HTTPError as h:
+ self.fatal('URL could not be fetched (' + str(res.status_code) + ' / ' + res.content + ')')
except Exception as x:
if not noLog:
- self.info("Unexpected exception occurred fetching: " + url + " (" + str(x) + ")")
+ try:
+ self.error("Unexpected exception (" + str(x) + ") occurred fetching: " + url, False)
+ self.error(traceback.format_exc(), False)
+ except BaseException as f:
+ return result
result['content'] = None
result['status'] = str(x)
if fatal:
self.fatal('URL could not be fetched (' + str(x) + ')')
+ frm = inspect.stack()[1]
+ mod = inspect.getmodule(frm[0])
+ m = mod.__name__
+ atime = time.time()
+ t = str(atime - btime)
+ self.info("Fetched data: " + str(len(result['content'] or '')) + " (" + url + "), took " + t + "s")
return result
# Check if wildcard DNS is enabled by looking up two random hostnames
@@ -1860,6 +1957,7 @@ class SpiderFootEvent(object):
sourceEvent = None
sourceEventHash = None
moduleDataSource = None
+ actualSource = None
__id = None
def __init__(self, eventType, data, module, sourceEvent,
@@ -1923,23 +2021,6 @@ class SpiderFootEvent(object):
self.sourceEventHash = srcHash
-# Override the default redirectors to re-use cookies
-class SmartRedirectHandler(urllib2.HTTPRedirectHandler):
- def http_error_301(self, req, fp, code, msg, headers):
- if "Set-Cookie" in headers:
- req.add_header('cookie', headers['Set-Cookie'])
- result = urllib2.HTTPRedirectHandler.http_error_301(
- self, req, fp, code, msg, headers)
- return result
-
- def http_error_302(self, req, fp, code, msg, headers):
- if "Set-Cookie" in headers:
- req.add_header('cookie', headers['Set-Cookie'])
- result = urllib2.HTTPRedirectHandler.http_error_302(
- self, req, fp, code, msg, headers)
- return result
-
-
"""
Public Suffix List module for Python.
See LICENSE.tp for applicable license.