SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.
Python attacksurfacecticybersecurityfootprintinghackinginformation-gatheringinformation-securityinfosecintelligence-gatheringosintosint-frameworkosint-reconnaissanceosint-toolpentestingpythonreconsecurity-toolsthreat-intelligencethreatintel
Various module bug fixes. (#1080)
Steve Micallefa4dfd2847230
diff --git a/modules/sfp_accounts.py b/modules/sfp_accounts.pyindex 6992a6b4..ea7d2d8a 100644--- a/modules/sfp_accounts.py+++ b/modules/sfp_accounts.py@@ -255,7 +255,7 @@ class sfp_accounts(SpiderFootPlugin):users.append(kw)- if eventName == "EMAILADDR":+ if eventName == "EMAILADDR" and self.opts['userfromemail']:name = eventData.split("@")[0].lower()users.append(name)diff --git a/modules/sfp_archiveorg.py b/modules/sfp_archiveorg.pyindex 7d4ed80d..d8ea5f4f 100644--- a/modules/sfp_archiveorg.py+++ b/modules/sfp_archiveorg.py@@ -77,11 +77,13 @@ class sfp_archiveorg(SpiderFootPlugin):results = NonefoundDates = list()+ errorState = Falsedef setup(self, sfc, userOpts=dict()):self.sf = sfcself.results = self.tempStorage()self.foundDates = list()+ self.errorState = Falsefor opt in list(userOpts.keys()):self.opts[opt] = userOpts[opt]@@ -108,6 +110,9 @@ class sfp_archiveorg(SpiderFootPlugin):srcModuleName = event.moduleeventData = event.data+ if self.errorState:+ return+self.sf.debug(f"Received event, {eventName}, from {srcModuleName}")if eventName == "INTERESTING_FILE" and not self.opts['intfiles']:@@ -135,7 +140,13 @@ class sfp_archiveorg(SpiderFootPlugin):self.results[eventData] = Truefor daysback in self.opts['farback'].split(","):- newDate = datetime.datetime.now() - datetime.timedelta(days=int(daysback))+ try:+ newDate = datetime.datetime.now() - datetime.timedelta(days=int(daysback))+ except Exception:+ self.sf.error("Unable to parse option for number of days back to search.")+ self.errorState = True+ return+maxDate = newDate.strftime("%Y%m%d")url = "https://archive.org/wayback/available?url=" + eventData + \diff --git a/modules/sfp_azureblobstorage.py b/modules/sfp_azureblobstorage.pyindex 764c7349..28d4ad25 100644--- a/modules/sfp_azureblobstorage.py+++ b/modules/sfp_azureblobstorage.py@@ -11,6 +11,7 @@# Licence: GPL# -------------------------------------------------------------------------------+import randomimport threadingimport time@@ -83,7 +84,8 @@ class sfp_azureblobstorage(SpiderFootPlugin):return Noneself.sf.info("Spawning thread to check bucket: " + site)- t.append(threading.Thread(name='thread_sfp_azureblobstorages_' + site,+ tname = str(random.SystemRandom().randint(0, 999999999))+ t.append(threading.Thread(name='thread_sfp_azureblobstorages_' + tname,target=self.checkSite, args=(site,)))t[i].start()i += 1diff --git a/modules/sfp_base64.py b/modules/sfp_base64.pyindex 9c5fe616..825e6fd8 100644--- a/modules/sfp_base64.py+++ b/modules/sfp_base64.py@@ -1,7 +1,7 @@# -*- coding: utf-8 -*-# -------------------------------------------------------------------------------# Name: sfp_base64-# Purpose: Identifies (best-effort) Base64-encoded strings in content and URLs.+# Purpose: Identifies (best-effort) Base64-encoded strings in URLs.## Author: Steve Micallef <[email protected]>#@@ -19,7 +19,7 @@ class sfp_base64(SpiderFootPlugin):meta = {'name': "Base64 Decoder",- 'summary': "Identify Base64-encoded strings in any content and URLs, often revealing interesting hidden information.",+ 'summary': "Identify Base64-encoded strings in URLs, often revealing interesting hidden information.",'flags': [""],'useCases': ["Investigate", "Passive"],'categories': ["Content Analysis"]@@ -44,7 +44,7 @@ class sfp_base64(SpiderFootPlugin):# What events is this module interested in for inputdef watchedEvents(self):- return ["LINKED_URL_INTERNAL", "TARGET_WEB_CONTENT"]+ return ["LINKED_URL_INTERNAL"]# What events this module producesdef producedEvents(self):diff --git a/modules/sfp_creditcard.py b/modules/sfp_creditcard.pyindex 60f4ed6e..362c494b 100644--- a/modules/sfp_creditcard.py+++ b/modules/sfp_creditcard.py@@ -19,7 +19,7 @@ class sfp_creditcard(SpiderFootPlugin):meta = {'name': "Credit Card Number Extractor",'summary': "Identify Credit Card Numbers in any data",- 'flags': [""],+ 'flags': ["errorprone"],'useCases': ["Footprint", "Investigate", "Passive"],'categories': ["Content Analysis"]}diff --git a/modules/sfp_crxcavator.py b/modules/sfp_crxcavator.pyindex 2986ff7a..260c61ce 100644--- a/modules/sfp_crxcavator.py+++ b/modules/sfp_crxcavator.py@@ -150,6 +150,9 @@ class sfp_crxcavator(SpiderFootPlugin):locations = list()for result in results:+ if not isinstance(result, dict):+ continue+extension_id = result.get('extension_id')if not extension_id:diff --git a/modules/sfp_digitaloceanspace.py b/modules/sfp_digitaloceanspace.pyindex da17d599..29b7d38c 100644--- a/modules/sfp_digitaloceanspace.py+++ b/modules/sfp_digitaloceanspace.py@@ -11,6 +11,7 @@# Licence: GPL# -------------------------------------------------------------------------------+import randomimport threadingimport time@@ -100,7 +101,8 @@ class sfp_digitaloceanspace(SpiderFootPlugin):return Noneself.sf.info("Spawning thread to check bucket: " + site)- t.append(threading.Thread(name='thread_sfp_digitaloceanspaces_' + site,+ tname = str(random.SystemRandom().randint(0, 999999999))+ t.append(threading.Thread(name='thread_sfp_digitaloceanspaces_' + tname,target=self.checkSite, args=(site,)))t[i].start()i += 1diff --git a/modules/sfp_dnsdb.py b/modules/sfp_dnsdb.pyindex 08a3a1af..e7bcff20 100644--- a/modules/sfp_dnsdb.py+++ b/modules/sfp_dnsdb.py@@ -89,6 +89,7 @@ class sfp_dnsdb(SpiderFootPlugin):"PROVIDER_MAIL","IP_ADDRESS","IPV6_ADDRESS",+ "CO_HOSTED_SITE",]def query(self, endpoint, queryType, query):diff --git a/modules/sfp_emailcrawlr.py b/modules/sfp_emailcrawlr.pyindex 6a46beea..f5a8514c 100644--- a/modules/sfp_emailcrawlr.py+++ b/modules/sfp_emailcrawlr.py@@ -81,7 +81,7 @@ class sfp_emailcrawlr(SpiderFootPlugin):# What events this module producesdef producedEvents(self):return ["RAW_RIR_DATA", "EMAILADDR", "EMAILADDR_GENERIC",- "PHONE_NUMBER", "GEOINFO", "HUMAN_NAME"]+ "PHONE_NUMBER", "GEOINFO"]# Query domain# https://emailcrawlr.com/docs@@ -197,7 +197,8 @@ class sfp_emailcrawlr(SpiderFootPlugin):if name:full_name = name.get('name')if full_name and len(full_name) > 3:- evt = SpiderFootEvent("HUMAN_NAME", full_name, self.__name__, event)+ evt = SpiderFootEvent("RAW_RIR_DATA", f"Possible full name: {full_name}",+ self.__name__, event)self.notifyListeners(evt)phone_numbers = res.get('numbers')diff --git a/modules/sfp_fringeproject.py b/modules/sfp_fringeproject.pyindex c2f8653e..950d27a1 100644--- a/modules/sfp_fringeproject.py+++ b/modules/sfp_fringeproject.py@@ -44,9 +44,11 @@ class sfp_fringeproject(SpiderFootPlugin):}opts = {+ 'verify': True}optdescs = {+ 'verify': "Verify hosts resolve before reporting them."}results = None@@ -64,7 +66,8 @@ class sfp_fringeproject(SpiderFootPlugin):def producedEvents(self):return ['INTERNET_NAME', 'LINKED_URL_INTERNAL', 'DOMAIN_NAME',- 'TCP_PORT_OPEN', 'SOFTWARE_USED', 'RAW_RIR_DATA']+ 'TCP_PORT_OPEN', 'SOFTWARE_USED', 'RAW_RIR_DATA',+ 'INTERNET_NAME_UNRESOLVED']def query(self, qry):params = {@@ -171,8 +174,13 @@ class sfp_fringeproject(SpiderFootPlugin):self.notifyListeners(evt)for host in set(hosts):- evt = SpiderFootEvent('INTERNET_NAME', host, self.__name__, event)+ if self.opts["verify"] and not self.sf.resolveHost(host):+ self.sf.debug(f"Host {host} could not be resolved.")+ evt = SpiderFootEvent("INTERNET_NAME_UNRESOLVED", host, self.__name__, event)+ else:+ evt = SpiderFootEvent("INTERNET_NAME", host, self.__name__, event)self.notifyListeners(evt)+if self.sf.isDomain(host, self.opts['_internettlds']):evt = SpiderFootEvent('DOMAIN_NAME', host, self.__name__, event)self.notifyListeners(evt)diff --git a/modules/sfp_googleobjectstorage.py b/modules/sfp_googleobjectstorage.pyindex b69ddaac..40c8a758 100644--- a/modules/sfp_googleobjectstorage.py+++ b/modules/sfp_googleobjectstorage.py@@ -11,6 +11,7 @@# Licence: GPL# -------------------------------------------------------------------------------+import randomimport threadingimport time@@ -98,7 +99,8 @@ class sfp_googleobjectstorage(SpiderFootPlugin):return Noneself.sf.info("Spawning thread to check bucket: " + site)- t.append(threading.Thread(name='thread_sfp_googleobjectstorage_' + site,+ tname = str(random.SystemRandom().randint(0, 999999999))+ t.append(threading.Thread(name='thread_sfp_googleobjectstorage_' + tname,target=self.checkSite, args=(site,)))t[i].start()i += 1diff --git a/modules/sfp_gravatar.py b/modules/sfp_gravatar.pyindex 81dc0362..042c3b62 100644--- a/modules/sfp_gravatar.py+++ b/modules/sfp_gravatar.py@@ -66,7 +66,7 @@ class sfp_gravatar(SpiderFootPlugin):# What events this module producesdef producedEvents(self):- return ['RAW_RIR_DATA', 'HUMAN_NAME', 'USERNAME',+ return ['RAW_RIR_DATA', 'USERNAME','EMAILADDR', 'EMAILADDR_GENERIC', 'PHONE_NUMBER', 'GEOINFO','ACCOUNT_EXTERNAL_OWNED', 'SOCIAL_MEDIA']@@ -137,8 +137,9 @@ class sfp_gravatar(SpiderFootPlugin):names = data.get('name')for name in names:- if name.get('formatted') is not None:- evt = SpiderFootEvent("HUMAN_NAME", name.get('formatted'), self.__name__, event)+ full_name = name.get('formatted')+ if full_name:+ evt = SpiderFootEvent("RAW_RIR_DATA", f"Possible full name: {full_name}", self.__name__, event)self.notifyListeners(evt)# TODO: re-enable once location validation is implementeddiff --git a/modules/sfp_iban.py b/modules/sfp_iban.pyindex 451e10f8..8b8362c2 100644--- a/modules/sfp_iban.py+++ b/modules/sfp_iban.py@@ -19,7 +19,7 @@ class sfp_iban(SpiderFootPlugin):meta = {'name': "IBAN Number Extractor",'summary': "Identify IBAN Numbers in any data",- 'flags': [""],+ 'flags': ["errorprone"],'useCases': ["Footprint", "Investigate", "Passive"],'categories': ["Content Analysis"]}diff --git a/modules/sfp_intelx.py b/modules/sfp_intelx.pyindex 3d3ee9ed..3cd6d6c3 100644--- a/modules/sfp_intelx.py+++ b/modules/sfp_intelx.py@@ -135,6 +135,11 @@ class sfp_intelx(SpiderFootPlugin):self.sf.info("No IntelligenceX info found for " + qry)return None+ if res['code'] == "402":+ self.sf.info("IntelligenceX credits expired.")+ self.errorState = True+ return None+try:ret = json.loads(res['content'])except Exception as e:@@ -157,6 +162,11 @@ class sfp_intelx(SpiderFootPlugin):self.sf.info("No IntelligenceX info found for results from " + qry)return None+ if res['code'] == "402":+ self.sf.info("IntelligenceX credits expired.")+ self.errorState = True+ return None+try:ret = json.loads(res['content'])except Exception as e:diff --git a/modules/sfp_jsonwhoiscom.py b/modules/sfp_jsonwhoiscom.pyindex 3a802cde..4b79b132 100644--- a/modules/sfp_jsonwhoiscom.py+++ b/modules/sfp_jsonwhoiscom.py@@ -80,7 +80,7 @@ class sfp_jsonwhoiscom(SpiderFootPlugin):def producedEvents(self):return ["RAW_RIR_DATA", "DOMAIN_REGISTRAR", "DOMAIN_WHOIS", "PROVIDER_DNS","EMAILADDR", "EMAILADDR_GENERIC", "PHONE_NUMBER", "PHYSICAL_ADDRESS",- "HUMAN_NAME", "AFFILIATE_DOMAIN_UNREGISTERED"]+ "AFFILIATE_DOMAIN_UNREGISTERED"]# Query domain# https://jsonwhois.com/docs@@ -260,7 +260,7 @@ class sfp_jsonwhoiscom(SpiderFootPlugin):self.notifyListeners(evt)for name in set(names):- evt = SpiderFootEvent("HUMAN_NAME", name, self.__name__, event)+ evt = SpiderFootEvent("RAW_RIR_DATA", f"Possible full name {name}", self.__name__, event)self.notifyListeners(evt)for phone in set(phones):diff --git a/modules/sfp_keybase.py b/modules/sfp_keybase.pyindex f0972e6d..e0634ed9 100644--- a/modules/sfp_keybase.py+++ b/modules/sfp_keybase.py@@ -67,8 +67,7 @@ class sfp_keybase(SpiderFootPlugin):def producedEvents(self):return ["RAW_RIR_DATA", "SOCIAL_MEDIA", "USERNAME",- "HUMAN_NAME", "GEOINFO", "BITCOIN_ADDRESS",- "PGP_KEY"+ "GEOINFO", "BITCOIN_ADDRESS", "PGP_KEY"]def queryUsername(self, qry):@@ -190,7 +189,8 @@ class sfp_keybase(SpiderFootPlugin):# Get and report full name of userfullName = profile.get('full_name')if fullName:- evt = SpiderFootEvent("HUMAN_NAME", fullName, self.__name__, event)+ evt = SpiderFootEvent("RAW_RIR_DATA", f"Possible full name: {fullName}",+ self.__name__, event)self.notifyListeners(evt)# Get and report location of userdiff --git a/modules/sfp_onyphe.py b/modules/sfp_onyphe.pyindex dcc6a169..441cd86d 100644--- a/modules/sfp_onyphe.py+++ b/modules/sfp_onyphe.py@@ -56,6 +56,8 @@ class sfp_onyphe(SpiderFootPlugin):"max_page": 10,"verify": True,"age_limit_days": 30,+ "cohostsamedomain": False,+ "maxcohost": 100,}optdescs = {"api_key": "Onyphe access token.",@@ -63,10 +65,13 @@ class sfp_onyphe(SpiderFootPlugin):"max_page": "Maximum number of pages to iterate through. Onyphe has a maximum of 1000 pages (10,000 results). Only matters for paid plans","verify": "Verify identified domains still resolve to the associated specified IP address.","age_limit_days": "Ignore any records older than this many days. 0 = unlimited.",+ "maxcohost": "Stop reporting co-hosted sites after this many are found, as it would likely indicate web hosting.",+ "cohostsamedomain": "Treat co-hosted sites on the same target domain as co-hosting?",}results = NoneerrorState = False+ cohostcount = 0def setup(self, sfc, userOpts=dict()):self.sf = sfc@@ -183,15 +188,33 @@ class sfp_onyphe(SpiderFootPlugin):domains.add(subDomain)for domain in domains:- if self.opts["verify"] and not self.sf.resolveHost(domain):- self.sf.debug(f"Host {domain} could not be resolved for {eventData}")- evt = SpiderFootEvent(- "INTERNET_NAME_UNRESOLVED", domain, self.__name__, event- )+ if self.getTarget().matches(domain, includeParents=True):+ if self.opts['verify'] and self.sf.resolveHost(domain):+ evt = SpiderFootEvent('INTERNET_NAME', domain, self.__name__, event)+ else:+ evt = SpiderFootEvent('INTERNET_NAME_UNRESOLVED', domain, self.__name__, event)self.notifyListeners(evt)- else:- evt = SpiderFootEvent("INTERNET_NAME", domain, self.__name__, event)++ if self.sf.isDomain(domain, self.opts['_internettlds']):+ evt = SpiderFootEvent('DOMAIN_NAME', domain, self.__name__, event)+ self.notifyListeners(evt)+ continue++ if self.cohostcount < self.opts['maxcohost']:+ if self.opts["verify"] and not self.sf.validateIP(domain, eventData):+ self.sf.debug("Host no longer resolves to our IP.")+ continue++ if not self.opts["cohostsamedomain"]:+ if self.getTarget().matches(domain, includeParents=True):+ self.sf.debug(+ "Skipping " + domain + " because it is on the same domain."+ )+ continue++ evt = SpiderFootEvent("CO_HOSTED_SITE", domain, self.__name__, event)self.notifyListeners(evt)+ self.cohostcount += 1def isFreshEnough(self, result):limit = self.opts["age_limit_days"]diff --git a/modules/sfp_riskiq.py b/modules/sfp_riskiq.pyindex 1e06333c..d997fd50 100644--- a/modules/sfp_riskiq.py+++ b/modules/sfp_riskiq.py@@ -91,7 +91,7 @@ class sfp_riskiq(SpiderFootPlugin):# What events this module producesdef producedEvents(self):return ["IP_ADDRESS", "INTERNET_NAME", "AFFILIATE_INTERNET_NAME",- "DOMAIN_NAME", "AFFILIATE_DOMAIN_NAME",+ "DOMAIN_NAME", "AFFILIATE_DOMAIN_NAME", "INTERNET_NAME_UNRESOLVED","CO_HOSTED_SITE", "NETBLOCK_OWNER"]def query(self, qry, qtype, opts=dict()):@@ -190,9 +190,14 @@ class sfp_riskiq(SpiderFootPlugin):if res['subjectCommonName'] == eventData:continueif self.getTarget().matches(res['subjectCommonName'], includeChildren=True):- e = SpiderFootEvent("INTERNET_NAME", res['subjectCommonName'],- self.__name__, event)+ if self.sf.resolveHost(res['subjectCommonName']):+ e = SpiderFootEvent("INTERNET_NAME", res['subjectCommonName'],+ self.__name__, event)+ else:+ e = SpiderFootEvent("INTERNET_NAME_UNRESOLVED", res['subjectCommonName'],+ self.__name__, event)self.notifyListeners(e)+if self.sf.isDomain(res['subjectCommonName'], self.opts['_internettlds']):e = SpiderFootEvent("DOMAIN_NAME", res['subjectCommonName'],self.__name__, event)@@ -257,8 +262,12 @@ class sfp_riskiq(SpiderFootPlugin):if not self.opts['cohostsamedomain']:if self.getTarget().matches(co, includeParents=True):- e = SpiderFootEvent("INTERNET_NAME", co, self.__name__, event)+ if self.sf.resolveHost(co):+ e = SpiderFootEvent("INTERNET_NAME", co, self.__name__, event)+ else:+ e = SpiderFootEvent("INTERNET_NAME_UNRESOLVED", co, self.__name__, event)self.notifyListeners(e)+if self.sf.isDomain(co, self.opts['_internettlds']):e = SpiderFootEvent("DOMAIN_NAME", co, self.__name__, event)self.notifyListeners(e)diff --git a/modules/sfp_s3bucket.py b/modules/sfp_s3bucket.pyindex 95f117d4..dbaf2895 100644--- a/modules/sfp_s3bucket.py+++ b/modules/sfp_s3bucket.py@@ -11,6 +11,7 @@# Licence: GPL# -------------------------------------------------------------------------------+import randomimport threadingimport time@@ -100,7 +101,8 @@ class sfp_s3bucket(SpiderFootPlugin):return Noneself.sf.info("Spawning thread to check bucket: " + site)- t.append(threading.Thread(name='thread_sfp_s3buckets_' + site,+ tname = str(random.SystemRandom().randint(0, 999999999))+ t.append(threading.Thread(name='thread_sfp_s3buckets_' + tname,target=self.checkSite, args=(site,)))t[i].start()i += 1diff --git a/modules/sfp_scylla.py b/modules/sfp_scylla.pyindex 7a555f04..2e4d7ae4 100644--- a/modules/sfp_scylla.py+++ b/modules/sfp_scylla.py@@ -9,7 +9,6 @@# Licence: GPL# --------------------------------------------------------------------------------import base64import jsonimport timeimport urllib.error@@ -78,16 +77,12 @@ class sfp_scylla(SpiderFootPlugin):def query(self, qry, per_page=20, start=0):params = {'q': 'Email:@' + qry.encode('raw_unicode_escape').decode("ascii", errors='replace'),- 'num': str(per_page),- 'from': str(start)+ 'size': str(per_page),+ 'start': str(start)}- b64_auth = base64.b64encode("sammy:BasicPassword!".encode("utf-8"))headers = {- 'Accept': 'application/json',- # Provided by @_hyp3ri0n on Twitter, owner of the service and granted- # permission to hard-code these.- 'Authorization': "Basic " + b64_auth.decode("utf-8")+ 'Accept': 'application/json'}res = self.sf.fetchUrl('https://scylla.sh/search?' + urllib.parse.urlencode(params),headers=headers,@@ -156,13 +151,13 @@ class sfp_scylla(SpiderFootPlugin):# evt = SpiderFootEvent('RAW_RIR_DATA', str(data), self.__name__, event)# self.notifyListeners(evt)- for result in data:- source = result.get('_source')+ for row in data:+ result = row.get('fields')- if not source:+ if not result:continue- email = source.get('Email')+ email = result.get('email')# A blank email result should not be possible, as we searched using the 'Email:' filterif not email:@@ -180,23 +175,18 @@ class sfp_scylla(SpiderFootPlugin):self.sf.debug("Skipped address: " + email)continue- breach = source.get('Domain')-- if not breach:- breach = 'Unknown'-+ breach = result.get('domain', 'Unknown')emails.append(email + " [" + breach + "]")-- pass_hash = source.get('PassHash')+ pass_hash = result.get('passhash')if pass_hash:- pass_salt = source.get('PassSalt')+ pass_salt = result.get('passsalt')if pass_salt:hashes.append(email + ':' + pass_hash + " (Salt: " + pass_salt + ") [" + breach + "]")else:hashes.append(email + ':' + pass_hash + " [" + breach + "]")- password = source.get('Password')+ password = result.get('password')if password:passwords.append(email + ':' + password + " [" + breach + "]")diff --git a/modules/sfp_tool_whatweb.py b/modules/sfp_tool_whatweb.pyindex 9e397d67..f402ab96 100644--- a/modules/sfp_tool_whatweb.py+++ b/modules/sfp_tool_whatweb.py@@ -152,9 +152,6 @@ class sfp_tool_whatweb(SpiderFootPlugin):if len(result_json) == 0:return- evt = SpiderFootEvent('RAW_RIR_DATA', str(result_json), self.__name__, event)- self.notifyListeners(evt)-blacklist = ['Country', 'IP','Script', 'Title',@@ -163,6 +160,7 @@ class sfp_tool_whatweb(SpiderFootPlugin):'X-UA-Compatible', 'X-Powered-By', 'X-Forwarded-For', 'X-Frame-Options', 'X-XSS-Protection']+ found = Falsefor result in result_json:plugin_matches = result.get('plugins')@@ -173,16 +171,23 @@ class sfp_tool_whatweb(SpiderFootPlugin):for w in plugin_matches.get('HTTPServer').get('string'):evt = SpiderFootEvent('WEBSERVER_BANNER', w, self.__name__, event)self.notifyListeners(evt)+ found = Trueif plugin_matches.get('X-Powered-By'):for w in plugin_matches.get('X-Powered-By').get('string'):evt = SpiderFootEvent('WEBSERVER_TECHNOLOGY', w, self.__name__, event)self.notifyListeners(evt)+ found = Truefor plugin in plugin_matches:if plugin in blacklist:continueevt = SpiderFootEvent('WEBSERVER_TECHNOLOGY', plugin, self.__name__, event)self.notifyListeners(evt)+ found = True++ if found:+ evt = SpiderFootEvent('RAW_RIR_DATA', str(result_json), self.__name__, event)+ self.notifyListeners(evt)# End of sfp_tool_whatweb classdiff --git a/modules/sfp_xforce.py b/modules/sfp_xforce.pyindex afb2020e..d0f011a4 100644--- a/modules/sfp_xforce.py+++ b/modules/sfp_xforce.py@@ -25,7 +25,7 @@ class sfp_xforce(SpiderFootPlugin):meta = {'name': "XForce Exchange",- 'summary': "Obtain information from IBM X-Force Exchange",+ 'summary': "Obtain IP reputation and passive DNS information from IBM X-Force Exchange",'flags': ["apikey"],'useCases': ["Investigate", "Passive"],'categories': ["Reputation Systems"],@@ -64,7 +64,9 @@ class sfp_xforce(SpiderFootPlugin):'subnetlookup': True,'maxsubnet': 24,'maxcohost': 100,- 'checkaffiliates': True+ 'cohostsamedomain': False,+ 'checkaffiliates': True,+ 'verify': True,}# Option descriptions@@ -77,7 +79,9 @@ class sfp_xforce(SpiderFootPlugin):'subnetlookup': "Look up all IPs on subnets which your target is a part of for blacklisting?",'maxsubnet': "If looking up subnets, the maximum subnet size to look up all the IPs within (CIDR value, 24 = /24, 16 = /16, etc.)",'maxcohost': "Stop reporting co-hosted sites after this many are found, as it would likely indicate web hosting.",- 'checkaffiliates': "Apply checks to affiliates?"+ "cohostsamedomain": "Treat co-hosted sites on the same target domain as co-hosting?",+ 'checkaffiliates': "Apply checks to affiliates?",+ 'verify': "Verify identified domains still resolve to the associated specified IP address.",}# Be sure to completely clear any class variables in setup()@@ -100,16 +104,13 @@ class sfp_xforce(SpiderFootPlugin):# What events is this module interested in for inputdef watchedEvents(self):- return ["IP_ADDRESS", "AFFILIATE_IPADDR", "INTERNET_NAME",- "CO_HOSTED_SITE", "NETBLOCK_OWNER", "NETBLOCK_MEMBER",- "AFFILIATE_INTERNET_NAME"]+ return ["IP_ADDRESS", "AFFILIATE_IPADDR",+ "NETBLOCK_OWNER", "NETBLOCK_MEMBER"]# What events this module producesdef producedEvents(self):- return ["MALICIOUS_IPADDR", "MALICIOUS_INTERNET_NAME",- "MALICIOUS_COHOST", "MALICIOUS_AFFILIATE_INTERNET_NAME",- "MALICIOUS_AFFILIATE_IPADDR", "MALICIOUS_NETBLOCK",- "CO_HOSTED_SITE"]+ return ["MALICIOUS_IPADDR", "CO_HOSTED_SITE",+ "MALICIOUS_AFFILIATE_IPADDR", "MALICIOUS_NETBLOCK"]def query(self, qry, querytype):if querytype not in ["ipr/malware", "ipr/history", "resolve"]:@@ -238,7 +239,6 @@ class sfp_xforce(SpiderFootPlugin):# For IP Addresses, do the additional passive DNS lookupif eventName == "IP_ADDRESS":- evtType = "CO_HOSTED_SITE"if self.cohostcount > self.opts['maxcohost']:return@@ -261,7 +261,17 @@ class sfp_xforce(SpiderFootPlugin):self.sf.debug("Record found but too old, skipping.")continueelse:- e = SpiderFootEvent(evtType, host, self.__name__, event)+ if not self.opts["cohostsamedomain"]:+ if self.getTarget().matches(host, includeParents=True):+ self.sf.debug(+ "Skipping " + host + " because it is on the same domain."+ )+ continue++ if self.opts['verify'] and not self.sf.resolveHost(host):+ continue++ e = SpiderFootEvent("CO_HOSTED_SITE", host, self.__name__, event)self.notifyListeners(e)self.cohostcount += 1@@ -273,12 +283,6 @@ class sfp_xforce(SpiderFootPlugin):evtType = 'MALICIOUS_IPADDR'if eventName == "AFFILIATE_IPADDR":evtType = 'MALICIOUS_AFFILIATE_IPADDR'- if eventName == "INTERNET_NAME":- evtType = "MALICIOUS_INTERNET_NAME"- if eventName == 'AFFILIATE_INTERNET_NAME':- evtType = 'MALICIOUS_AFFILIATE_INTERNET_NAME'- if eventName == 'CO_HOSTED_SITE':- evtType = 'MALICIOUS_COHOST'rec = self.query(addr, "ipr/history")if rec is not None:@@ -313,10 +317,6 @@ class sfp_xforce(SpiderFootPlugin):e = SpiderFootEvent(evtType, entry, self.__name__, event)self.notifyListeners(e)- # ipr/malware doesn't support hostnames- if eventName in ["CO_HOSTED_SITE", "INTERNET_NAME", "AFFILIATE_INTERNET_NAME"]:- continue-rec = self.query(addr, "ipr/malware")if rec is not None:rec_malware = rec.get("malware", list())diff --git a/test/unit/modules/test_sfp_base64.py b/test/unit/modules/test_sfp_base64.pyindex e9ca02b6..28f051d1 100644--- a/test/unit/modules/test_sfp_base64.py+++ b/test/unit/modules/test_sfp_base64.py@@ -54,7 +54,7 @@ class TestModuleBase64(unittest.TestCase):module = sfp_base64()self.assertIsInstance(module.producedEvents(), list)- def test_handleEvent_event_data_containing_base64_string_should_return_event(self):+ def test_handleEvent_event_data_url_containing_base64_string_should_return_event(self):sf = SpiderFoot(self.default_options)module = sfp_base64()@@ -77,7 +77,7 @@ class TestModuleBase64(unittest.TestCase):module.notifyListeners = new_notifyListeners.__get__(module, sfp_base64)event_type = 'ROOT'- event_data = 'example data U3BpZGVyRm9vdA== example data'+ event_data = 'https://someurl/?somevar=example%20data%20U3BpZGVyRm9vdA%3d%3d%20example%20data'event_module = ''source_event = ''