Skip to content

SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.

Various module bug fixes. (#1080)

Steve Micallefa4dfd2847230
diff --git a/modules/sfp_accounts.py b/modules/sfp_accounts.py
index 6992a6b4..ea7d2d8a 100644
--- a/modules/sfp_accounts.py
+++ b/modules/sfp_accounts.py
@@ -255,7 +255,7 @@ class sfp_accounts(SpiderFootPlugin):
users.append(kw)
- if eventName == "EMAILADDR":
+ if eventName == "EMAILADDR" and self.opts['userfromemail']:
name = eventData.split("@")[0].lower()
users.append(name)
diff --git a/modules/sfp_archiveorg.py b/modules/sfp_archiveorg.py
index 7d4ed80d..d8ea5f4f 100644
--- a/modules/sfp_archiveorg.py
+++ b/modules/sfp_archiveorg.py
@@ -77,11 +77,13 @@ class sfp_archiveorg(SpiderFootPlugin):
results = None
foundDates = list()
+ errorState = False
def setup(self, sfc, userOpts=dict()):
self.sf = sfc
self.results = self.tempStorage()
self.foundDates = list()
+ self.errorState = False
for opt in list(userOpts.keys()):
self.opts[opt] = userOpts[opt]
@@ -108,6 +110,9 @@ class sfp_archiveorg(SpiderFootPlugin):
srcModuleName = event.module
eventData = event.data
+ if self.errorState:
+ return
+
self.sf.debug(f"Received event, {eventName}, from {srcModuleName}")
if eventName == "INTERESTING_FILE" and not self.opts['intfiles']:
@@ -135,7 +140,13 @@ class sfp_archiveorg(SpiderFootPlugin):
self.results[eventData] = True
for daysback in self.opts['farback'].split(","):
- newDate = datetime.datetime.now() - datetime.timedelta(days=int(daysback))
+ try:
+ newDate = datetime.datetime.now() - datetime.timedelta(days=int(daysback))
+ except Exception:
+ self.sf.error("Unable to parse option for number of days back to search.")
+ self.errorState = True
+ return
+
maxDate = newDate.strftime("%Y%m%d")
url = "https://archive.org/wayback/available?url=" + eventData + \
diff --git a/modules/sfp_azureblobstorage.py b/modules/sfp_azureblobstorage.py
index 764c7349..28d4ad25 100644
--- a/modules/sfp_azureblobstorage.py
+++ b/modules/sfp_azureblobstorage.py
@@ -11,6 +11,7 @@
# Licence: GPL
# -------------------------------------------------------------------------------
+import random
import threading
import time
@@ -83,7 +84,8 @@ class sfp_azureblobstorage(SpiderFootPlugin):
return None
self.sf.info("Spawning thread to check bucket: " + site)
- t.append(threading.Thread(name='thread_sfp_azureblobstorages_' + site,
+ tname = str(random.SystemRandom().randint(0, 999999999))
+ t.append(threading.Thread(name='thread_sfp_azureblobstorages_' + tname,
target=self.checkSite, args=(site,)))
t[i].start()
i += 1
diff --git a/modules/sfp_base64.py b/modules/sfp_base64.py
index 9c5fe616..825e6fd8 100644
--- a/modules/sfp_base64.py
+++ b/modules/sfp_base64.py
@@ -1,7 +1,7 @@
# -*- coding: utf-8 -*-
# -------------------------------------------------------------------------------
# Name: sfp_base64
-# Purpose: Identifies (best-effort) Base64-encoded strings in content and URLs.
+# Purpose: Identifies (best-effort) Base64-encoded strings in URLs.
#
# Author: Steve Micallef <[email protected]>
#
@@ -19,7 +19,7 @@ class sfp_base64(SpiderFootPlugin):
meta = {
'name': "Base64 Decoder",
- 'summary': "Identify Base64-encoded strings in any content and URLs, often revealing interesting hidden information.",
+ 'summary': "Identify Base64-encoded strings in URLs, often revealing interesting hidden information.",
'flags': [""],
'useCases': ["Investigate", "Passive"],
'categories': ["Content Analysis"]
@@ -44,7 +44,7 @@ class sfp_base64(SpiderFootPlugin):
# What events is this module interested in for input
def watchedEvents(self):
- return ["LINKED_URL_INTERNAL", "TARGET_WEB_CONTENT"]
+ return ["LINKED_URL_INTERNAL"]
# What events this module produces
def producedEvents(self):
diff --git a/modules/sfp_creditcard.py b/modules/sfp_creditcard.py
index 60f4ed6e..362c494b 100644
--- a/modules/sfp_creditcard.py
+++ b/modules/sfp_creditcard.py
@@ -19,7 +19,7 @@ class sfp_creditcard(SpiderFootPlugin):
meta = {
'name': "Credit Card Number Extractor",
'summary': "Identify Credit Card Numbers in any data",
- 'flags': [""],
+ 'flags': ["errorprone"],
'useCases': ["Footprint", "Investigate", "Passive"],
'categories': ["Content Analysis"]
}
diff --git a/modules/sfp_crxcavator.py b/modules/sfp_crxcavator.py
index 2986ff7a..260c61ce 100644
--- a/modules/sfp_crxcavator.py
+++ b/modules/sfp_crxcavator.py
@@ -150,6 +150,9 @@ class sfp_crxcavator(SpiderFootPlugin):
locations = list()
for result in results:
+ if not isinstance(result, dict):
+ continue
+
extension_id = result.get('extension_id')
if not extension_id:
diff --git a/modules/sfp_digitaloceanspace.py b/modules/sfp_digitaloceanspace.py
index da17d599..29b7d38c 100644
--- a/modules/sfp_digitaloceanspace.py
+++ b/modules/sfp_digitaloceanspace.py
@@ -11,6 +11,7 @@
# Licence: GPL
# -------------------------------------------------------------------------------
+import random
import threading
import time
@@ -100,7 +101,8 @@ class sfp_digitaloceanspace(SpiderFootPlugin):
return None
self.sf.info("Spawning thread to check bucket: " + site)
- t.append(threading.Thread(name='thread_sfp_digitaloceanspaces_' + site,
+ tname = str(random.SystemRandom().randint(0, 999999999))
+ t.append(threading.Thread(name='thread_sfp_digitaloceanspaces_' + tname,
target=self.checkSite, args=(site,)))
t[i].start()
i += 1
diff --git a/modules/sfp_dnsdb.py b/modules/sfp_dnsdb.py
index 08a3a1af..e7bcff20 100644
--- a/modules/sfp_dnsdb.py
+++ b/modules/sfp_dnsdb.py
@@ -89,6 +89,7 @@ class sfp_dnsdb(SpiderFootPlugin):
"PROVIDER_MAIL",
"IP_ADDRESS",
"IPV6_ADDRESS",
+ "CO_HOSTED_SITE",
]
def query(self, endpoint, queryType, query):
diff --git a/modules/sfp_emailcrawlr.py b/modules/sfp_emailcrawlr.py
index 6a46beea..f5a8514c 100644
--- a/modules/sfp_emailcrawlr.py
+++ b/modules/sfp_emailcrawlr.py
@@ -81,7 +81,7 @@ class sfp_emailcrawlr(SpiderFootPlugin):
# What events this module produces
def producedEvents(self):
return ["RAW_RIR_DATA", "EMAILADDR", "EMAILADDR_GENERIC",
- "PHONE_NUMBER", "GEOINFO", "HUMAN_NAME"]
+ "PHONE_NUMBER", "GEOINFO"]
# Query domain
# https://emailcrawlr.com/docs
@@ -197,7 +197,8 @@ class sfp_emailcrawlr(SpiderFootPlugin):
if name:
full_name = name.get('name')
if full_name and len(full_name) > 3:
- evt = SpiderFootEvent("HUMAN_NAME", full_name, self.__name__, event)
+ evt = SpiderFootEvent("RAW_RIR_DATA", f"Possible full name: {full_name}",
+ self.__name__, event)
self.notifyListeners(evt)
phone_numbers = res.get('numbers')
diff --git a/modules/sfp_fringeproject.py b/modules/sfp_fringeproject.py
index c2f8653e..950d27a1 100644
--- a/modules/sfp_fringeproject.py
+++ b/modules/sfp_fringeproject.py
@@ -44,9 +44,11 @@ class sfp_fringeproject(SpiderFootPlugin):
}
opts = {
+ 'verify': True
}
optdescs = {
+ 'verify': "Verify hosts resolve before reporting them."
}
results = None
@@ -64,7 +66,8 @@ class sfp_fringeproject(SpiderFootPlugin):
def producedEvents(self):
return ['INTERNET_NAME', 'LINKED_URL_INTERNAL', 'DOMAIN_NAME',
- 'TCP_PORT_OPEN', 'SOFTWARE_USED', 'RAW_RIR_DATA']
+ 'TCP_PORT_OPEN', 'SOFTWARE_USED', 'RAW_RIR_DATA',
+ 'INTERNET_NAME_UNRESOLVED']
def query(self, qry):
params = {
@@ -171,8 +174,13 @@ class sfp_fringeproject(SpiderFootPlugin):
self.notifyListeners(evt)
for host in set(hosts):
- evt = SpiderFootEvent('INTERNET_NAME', host, self.__name__, event)
+ if self.opts["verify"] and not self.sf.resolveHost(host):
+ self.sf.debug(f"Host {host} could not be resolved.")
+ evt = SpiderFootEvent("INTERNET_NAME_UNRESOLVED", host, self.__name__, event)
+ else:
+ evt = SpiderFootEvent("INTERNET_NAME", host, self.__name__, event)
self.notifyListeners(evt)
+
if self.sf.isDomain(host, self.opts['_internettlds']):
evt = SpiderFootEvent('DOMAIN_NAME', host, self.__name__, event)
self.notifyListeners(evt)
diff --git a/modules/sfp_googleobjectstorage.py b/modules/sfp_googleobjectstorage.py
index b69ddaac..40c8a758 100644
--- a/modules/sfp_googleobjectstorage.py
+++ b/modules/sfp_googleobjectstorage.py
@@ -11,6 +11,7 @@
# Licence: GPL
# -------------------------------------------------------------------------------
+import random
import threading
import time
@@ -98,7 +99,8 @@ class sfp_googleobjectstorage(SpiderFootPlugin):
return None
self.sf.info("Spawning thread to check bucket: " + site)
- t.append(threading.Thread(name='thread_sfp_googleobjectstorage_' + site,
+ tname = str(random.SystemRandom().randint(0, 999999999))
+ t.append(threading.Thread(name='thread_sfp_googleobjectstorage_' + tname,
target=self.checkSite, args=(site,)))
t[i].start()
i += 1
diff --git a/modules/sfp_gravatar.py b/modules/sfp_gravatar.py
index 81dc0362..042c3b62 100644
--- a/modules/sfp_gravatar.py
+++ b/modules/sfp_gravatar.py
@@ -66,7 +66,7 @@ class sfp_gravatar(SpiderFootPlugin):
# What events this module produces
def producedEvents(self):
- return ['RAW_RIR_DATA', 'HUMAN_NAME', 'USERNAME',
+ return ['RAW_RIR_DATA', 'USERNAME',
'EMAILADDR', 'EMAILADDR_GENERIC', 'PHONE_NUMBER', 'GEOINFO',
'ACCOUNT_EXTERNAL_OWNED', 'SOCIAL_MEDIA']
@@ -137,8 +137,9 @@ class sfp_gravatar(SpiderFootPlugin):
names = data.get('name')
for name in names:
- if name.get('formatted') is not None:
- evt = SpiderFootEvent("HUMAN_NAME", name.get('formatted'), self.__name__, event)
+ full_name = name.get('formatted')
+ if full_name:
+ evt = SpiderFootEvent("RAW_RIR_DATA", f"Possible full name: {full_name}", self.__name__, event)
self.notifyListeners(evt)
# TODO: re-enable once location validation is implemented
diff --git a/modules/sfp_iban.py b/modules/sfp_iban.py
index 451e10f8..8b8362c2 100644
--- a/modules/sfp_iban.py
+++ b/modules/sfp_iban.py
@@ -19,7 +19,7 @@ class sfp_iban(SpiderFootPlugin):
meta = {
'name': "IBAN Number Extractor",
'summary': "Identify IBAN Numbers in any data",
- 'flags': [""],
+ 'flags': ["errorprone"],
'useCases': ["Footprint", "Investigate", "Passive"],
'categories': ["Content Analysis"]
}
diff --git a/modules/sfp_intelx.py b/modules/sfp_intelx.py
index 3d3ee9ed..3cd6d6c3 100644
--- a/modules/sfp_intelx.py
+++ b/modules/sfp_intelx.py
@@ -135,6 +135,11 @@ class sfp_intelx(SpiderFootPlugin):
self.sf.info("No IntelligenceX info found for " + qry)
return None
+ if res['code'] == "402":
+ self.sf.info("IntelligenceX credits expired.")
+ self.errorState = True
+ return None
+
try:
ret = json.loads(res['content'])
except Exception as e:
@@ -157,6 +162,11 @@ class sfp_intelx(SpiderFootPlugin):
self.sf.info("No IntelligenceX info found for results from " + qry)
return None
+ if res['code'] == "402":
+ self.sf.info("IntelligenceX credits expired.")
+ self.errorState = True
+ return None
+
try:
ret = json.loads(res['content'])
except Exception as e:
diff --git a/modules/sfp_jsonwhoiscom.py b/modules/sfp_jsonwhoiscom.py
index 3a802cde..4b79b132 100644
--- a/modules/sfp_jsonwhoiscom.py
+++ b/modules/sfp_jsonwhoiscom.py
@@ -80,7 +80,7 @@ class sfp_jsonwhoiscom(SpiderFootPlugin):
def producedEvents(self):
return ["RAW_RIR_DATA", "DOMAIN_REGISTRAR", "DOMAIN_WHOIS", "PROVIDER_DNS",
"EMAILADDR", "EMAILADDR_GENERIC", "PHONE_NUMBER", "PHYSICAL_ADDRESS",
- "HUMAN_NAME", "AFFILIATE_DOMAIN_UNREGISTERED"]
+ "AFFILIATE_DOMAIN_UNREGISTERED"]
# Query domain
# https://jsonwhois.com/docs
@@ -260,7 +260,7 @@ class sfp_jsonwhoiscom(SpiderFootPlugin):
self.notifyListeners(evt)
for name in set(names):
- evt = SpiderFootEvent("HUMAN_NAME", name, self.__name__, event)
+ evt = SpiderFootEvent("RAW_RIR_DATA", f"Possible full name {name}", self.__name__, event)
self.notifyListeners(evt)
for phone in set(phones):
diff --git a/modules/sfp_keybase.py b/modules/sfp_keybase.py
index f0972e6d..e0634ed9 100644
--- a/modules/sfp_keybase.py
+++ b/modules/sfp_keybase.py
@@ -67,8 +67,7 @@ class sfp_keybase(SpiderFootPlugin):
def producedEvents(self):
return [
"RAW_RIR_DATA", "SOCIAL_MEDIA", "USERNAME",
- "HUMAN_NAME", "GEOINFO", "BITCOIN_ADDRESS",
- "PGP_KEY"
+ "GEOINFO", "BITCOIN_ADDRESS", "PGP_KEY"
]
def queryUsername(self, qry):
@@ -190,7 +189,8 @@ class sfp_keybase(SpiderFootPlugin):
# Get and report full name of user
fullName = profile.get('full_name')
if fullName:
- evt = SpiderFootEvent("HUMAN_NAME", fullName, self.__name__, event)
+ evt = SpiderFootEvent("RAW_RIR_DATA", f"Possible full name: {fullName}",
+ self.__name__, event)
self.notifyListeners(evt)
# Get and report location of user
diff --git a/modules/sfp_onyphe.py b/modules/sfp_onyphe.py
index dcc6a169..441cd86d 100644
--- a/modules/sfp_onyphe.py
+++ b/modules/sfp_onyphe.py
@@ -56,6 +56,8 @@ class sfp_onyphe(SpiderFootPlugin):
"max_page": 10,
"verify": True,
"age_limit_days": 30,
+ "cohostsamedomain": False,
+ "maxcohost": 100,
}
optdescs = {
"api_key": "Onyphe access token.",
@@ -63,10 +65,13 @@ class sfp_onyphe(SpiderFootPlugin):
"max_page": "Maximum number of pages to iterate through. Onyphe has a maximum of 1000 pages (10,000 results). Only matters for paid plans",
"verify": "Verify identified domains still resolve to the associated specified IP address.",
"age_limit_days": "Ignore any records older than this many days. 0 = unlimited.",
+ "maxcohost": "Stop reporting co-hosted sites after this many are found, as it would likely indicate web hosting.",
+ "cohostsamedomain": "Treat co-hosted sites on the same target domain as co-hosting?",
}
results = None
errorState = False
+ cohostcount = 0
def setup(self, sfc, userOpts=dict()):
self.sf = sfc
@@ -183,15 +188,33 @@ class sfp_onyphe(SpiderFootPlugin):
domains.add(subDomain)
for domain in domains:
- if self.opts["verify"] and not self.sf.resolveHost(domain):
- self.sf.debug(f"Host {domain} could not be resolved for {eventData}")
- evt = SpiderFootEvent(
- "INTERNET_NAME_UNRESOLVED", domain, self.__name__, event
- )
+ if self.getTarget().matches(domain, includeParents=True):
+ if self.opts['verify'] and self.sf.resolveHost(domain):
+ evt = SpiderFootEvent('INTERNET_NAME', domain, self.__name__, event)
+ else:
+ evt = SpiderFootEvent('INTERNET_NAME_UNRESOLVED', domain, self.__name__, event)
self.notifyListeners(evt)
- else:
- evt = SpiderFootEvent("INTERNET_NAME", domain, self.__name__, event)
+
+ if self.sf.isDomain(domain, self.opts['_internettlds']):
+ evt = SpiderFootEvent('DOMAIN_NAME', domain, self.__name__, event)
+ self.notifyListeners(evt)
+ continue
+
+ if self.cohostcount < self.opts['maxcohost']:
+ if self.opts["verify"] and not self.sf.validateIP(domain, eventData):
+ self.sf.debug("Host no longer resolves to our IP.")
+ continue
+
+ if not self.opts["cohostsamedomain"]:
+ if self.getTarget().matches(domain, includeParents=True):
+ self.sf.debug(
+ "Skipping " + domain + " because it is on the same domain."
+ )
+ continue
+
+ evt = SpiderFootEvent("CO_HOSTED_SITE", domain, self.__name__, event)
self.notifyListeners(evt)
+ self.cohostcount += 1
def isFreshEnough(self, result):
limit = self.opts["age_limit_days"]
diff --git a/modules/sfp_riskiq.py b/modules/sfp_riskiq.py
index 1e06333c..d997fd50 100644
--- a/modules/sfp_riskiq.py
+++ b/modules/sfp_riskiq.py
@@ -91,7 +91,7 @@ class sfp_riskiq(SpiderFootPlugin):
# What events this module produces
def producedEvents(self):
return ["IP_ADDRESS", "INTERNET_NAME", "AFFILIATE_INTERNET_NAME",
- "DOMAIN_NAME", "AFFILIATE_DOMAIN_NAME",
+ "DOMAIN_NAME", "AFFILIATE_DOMAIN_NAME", "INTERNET_NAME_UNRESOLVED",
"CO_HOSTED_SITE", "NETBLOCK_OWNER"]
def query(self, qry, qtype, opts=dict()):
@@ -190,9 +190,14 @@ class sfp_riskiq(SpiderFootPlugin):
if res['subjectCommonName'] == eventData:
continue
if self.getTarget().matches(res['subjectCommonName'], includeChildren=True):
- e = SpiderFootEvent("INTERNET_NAME", res['subjectCommonName'],
- self.__name__, event)
+ if self.sf.resolveHost(res['subjectCommonName']):
+ e = SpiderFootEvent("INTERNET_NAME", res['subjectCommonName'],
+ self.__name__, event)
+ else:
+ e = SpiderFootEvent("INTERNET_NAME_UNRESOLVED", res['subjectCommonName'],
+ self.__name__, event)
self.notifyListeners(e)
+
if self.sf.isDomain(res['subjectCommonName'], self.opts['_internettlds']):
e = SpiderFootEvent("DOMAIN_NAME", res['subjectCommonName'],
self.__name__, event)
@@ -257,8 +262,12 @@ class sfp_riskiq(SpiderFootPlugin):
if not self.opts['cohostsamedomain']:
if self.getTarget().matches(co, includeParents=True):
- e = SpiderFootEvent("INTERNET_NAME", co, self.__name__, event)
+ if self.sf.resolveHost(co):
+ e = SpiderFootEvent("INTERNET_NAME", co, self.__name__, event)
+ else:
+ e = SpiderFootEvent("INTERNET_NAME_UNRESOLVED", co, self.__name__, event)
self.notifyListeners(e)
+
if self.sf.isDomain(co, self.opts['_internettlds']):
e = SpiderFootEvent("DOMAIN_NAME", co, self.__name__, event)
self.notifyListeners(e)
diff --git a/modules/sfp_s3bucket.py b/modules/sfp_s3bucket.py
index 95f117d4..dbaf2895 100644
--- a/modules/sfp_s3bucket.py
+++ b/modules/sfp_s3bucket.py
@@ -11,6 +11,7 @@
# Licence: GPL
# -------------------------------------------------------------------------------
+import random
import threading
import time
@@ -100,7 +101,8 @@ class sfp_s3bucket(SpiderFootPlugin):
return None
self.sf.info("Spawning thread to check bucket: " + site)
- t.append(threading.Thread(name='thread_sfp_s3buckets_' + site,
+ tname = str(random.SystemRandom().randint(0, 999999999))
+ t.append(threading.Thread(name='thread_sfp_s3buckets_' + tname,
target=self.checkSite, args=(site,)))
t[i].start()
i += 1
diff --git a/modules/sfp_scylla.py b/modules/sfp_scylla.py
index 7a555f04..2e4d7ae4 100644
--- a/modules/sfp_scylla.py
+++ b/modules/sfp_scylla.py
@@ -9,7 +9,6 @@
# Licence: GPL
# -------------------------------------------------------------------------------
-import base64
import json
import time
import urllib.error
@@ -78,16 +77,12 @@ class sfp_scylla(SpiderFootPlugin):
def query(self, qry, per_page=20, start=0):
params = {
'q': 'Email:@' + qry.encode('raw_unicode_escape').decode("ascii", errors='replace'),
- 'num': str(per_page),
- 'from': str(start)
+ 'size': str(per_page),
+ 'start': str(start)
}
- b64_auth = base64.b64encode("sammy:BasicPassword!".encode("utf-8"))
headers = {
- 'Accept': 'application/json',
- # Provided by @_hyp3ri0n on Twitter, owner of the service and granted
- # permission to hard-code these.
- 'Authorization': "Basic " + b64_auth.decode("utf-8")
+ 'Accept': 'application/json'
}
res = self.sf.fetchUrl('https://scylla.sh/search?' + urllib.parse.urlencode(params),
headers=headers,
@@ -156,13 +151,13 @@ class sfp_scylla(SpiderFootPlugin):
# evt = SpiderFootEvent('RAW_RIR_DATA', str(data), self.__name__, event)
# self.notifyListeners(evt)
- for result in data:
- source = result.get('_source')
+ for row in data:
+ result = row.get('fields')
- if not source:
+ if not result:
continue
- email = source.get('Email')
+ email = result.get('email')
# A blank email result should not be possible, as we searched using the 'Email:' filter
if not email:
@@ -180,23 +175,18 @@ class sfp_scylla(SpiderFootPlugin):
self.sf.debug("Skipped address: " + email)
continue
- breach = source.get('Domain')
-
- if not breach:
- breach = 'Unknown'
-
+ breach = result.get('domain', 'Unknown')
emails.append(email + " [" + breach + "]")
-
- pass_hash = source.get('PassHash')
+ pass_hash = result.get('passhash')
if pass_hash:
- pass_salt = source.get('PassSalt')
+ pass_salt = result.get('passsalt')
if pass_salt:
hashes.append(email + ':' + pass_hash + " (Salt: " + pass_salt + ") [" + breach + "]")
else:
hashes.append(email + ':' + pass_hash + " [" + breach + "]")
- password = source.get('Password')
+ password = result.get('password')
if password:
passwords.append(email + ':' + password + " [" + breach + "]")
diff --git a/modules/sfp_tool_whatweb.py b/modules/sfp_tool_whatweb.py
index 9e397d67..f402ab96 100644
--- a/modules/sfp_tool_whatweb.py
+++ b/modules/sfp_tool_whatweb.py
@@ -152,9 +152,6 @@ class sfp_tool_whatweb(SpiderFootPlugin):
if len(result_json) == 0:
return
- evt = SpiderFootEvent('RAW_RIR_DATA', str(result_json), self.__name__, event)
- self.notifyListeners(evt)
-
blacklist = [
'Country', 'IP',
'Script', 'Title',
@@ -163,6 +160,7 @@ class sfp_tool_whatweb(SpiderFootPlugin):
'X-UA-Compatible', 'X-Powered-By', 'X-Forwarded-For', 'X-Frame-Options', 'X-XSS-Protection'
]
+ found = False
for result in result_json:
plugin_matches = result.get('plugins')
@@ -173,16 +171,23 @@ class sfp_tool_whatweb(SpiderFootPlugin):
for w in plugin_matches.get('HTTPServer').get('string'):
evt = SpiderFootEvent('WEBSERVER_BANNER', w, self.__name__, event)
self.notifyListeners(evt)
+ found = True
if plugin_matches.get('X-Powered-By'):
for w in plugin_matches.get('X-Powered-By').get('string'):
evt = SpiderFootEvent('WEBSERVER_TECHNOLOGY', w, self.__name__, event)
self.notifyListeners(evt)
+ found = True
for plugin in plugin_matches:
if plugin in blacklist:
continue
evt = SpiderFootEvent('WEBSERVER_TECHNOLOGY', plugin, self.__name__, event)
self.notifyListeners(evt)
+ found = True
+
+ if found:
+ evt = SpiderFootEvent('RAW_RIR_DATA', str(result_json), self.__name__, event)
+ self.notifyListeners(evt)
# End of sfp_tool_whatweb class
diff --git a/modules/sfp_xforce.py b/modules/sfp_xforce.py
index afb2020e..d0f011a4 100644
--- a/modules/sfp_xforce.py
+++ b/modules/sfp_xforce.py
@@ -25,7 +25,7 @@ class sfp_xforce(SpiderFootPlugin):
meta = {
'name': "XForce Exchange",
- 'summary': "Obtain information from IBM X-Force Exchange",
+ 'summary': "Obtain IP reputation and passive DNS information from IBM X-Force Exchange",
'flags': ["apikey"],
'useCases': ["Investigate", "Passive"],
'categories': ["Reputation Systems"],
@@ -64,7 +64,9 @@ class sfp_xforce(SpiderFootPlugin):
'subnetlookup': True,
'maxsubnet': 24,
'maxcohost': 100,
- 'checkaffiliates': True
+ 'cohostsamedomain': False,
+ 'checkaffiliates': True,
+ 'verify': True,
}
# Option descriptions
@@ -77,7 +79,9 @@ class sfp_xforce(SpiderFootPlugin):
'subnetlookup': "Look up all IPs on subnets which your target is a part of for blacklisting?",
'maxsubnet': "If looking up subnets, the maximum subnet size to look up all the IPs within (CIDR value, 24 = /24, 16 = /16, etc.)",
'maxcohost': "Stop reporting co-hosted sites after this many are found, as it would likely indicate web hosting.",
- 'checkaffiliates': "Apply checks to affiliates?"
+ "cohostsamedomain": "Treat co-hosted sites on the same target domain as co-hosting?",
+ 'checkaffiliates': "Apply checks to affiliates?",
+ 'verify': "Verify identified domains still resolve to the associated specified IP address.",
}
# Be sure to completely clear any class variables in setup()
@@ -100,16 +104,13 @@ class sfp_xforce(SpiderFootPlugin):
# What events is this module interested in for input
def watchedEvents(self):
- return ["IP_ADDRESS", "AFFILIATE_IPADDR", "INTERNET_NAME",
- "CO_HOSTED_SITE", "NETBLOCK_OWNER", "NETBLOCK_MEMBER",
- "AFFILIATE_INTERNET_NAME"]
+ return ["IP_ADDRESS", "AFFILIATE_IPADDR",
+ "NETBLOCK_OWNER", "NETBLOCK_MEMBER"]
# What events this module produces
def producedEvents(self):
- return ["MALICIOUS_IPADDR", "MALICIOUS_INTERNET_NAME",
- "MALICIOUS_COHOST", "MALICIOUS_AFFILIATE_INTERNET_NAME",
- "MALICIOUS_AFFILIATE_IPADDR", "MALICIOUS_NETBLOCK",
- "CO_HOSTED_SITE"]
+ return ["MALICIOUS_IPADDR", "CO_HOSTED_SITE",
+ "MALICIOUS_AFFILIATE_IPADDR", "MALICIOUS_NETBLOCK"]
def query(self, qry, querytype):
if querytype not in ["ipr/malware", "ipr/history", "resolve"]:
@@ -238,7 +239,6 @@ class sfp_xforce(SpiderFootPlugin):
# For IP Addresses, do the additional passive DNS lookup
if eventName == "IP_ADDRESS":
- evtType = "CO_HOSTED_SITE"
if self.cohostcount > self.opts['maxcohost']:
return
@@ -261,7 +261,17 @@ class sfp_xforce(SpiderFootPlugin):
self.sf.debug("Record found but too old, skipping.")
continue
else:
- e = SpiderFootEvent(evtType, host, self.__name__, event)
+ if not self.opts["cohostsamedomain"]:
+ if self.getTarget().matches(host, includeParents=True):
+ self.sf.debug(
+ "Skipping " + host + " because it is on the same domain."
+ )
+ continue
+
+ if self.opts['verify'] and not self.sf.resolveHost(host):
+ continue
+
+ e = SpiderFootEvent("CO_HOSTED_SITE", host, self.__name__, event)
self.notifyListeners(e)
self.cohostcount += 1
@@ -273,12 +283,6 @@ class sfp_xforce(SpiderFootPlugin):
evtType = 'MALICIOUS_IPADDR'
if eventName == "AFFILIATE_IPADDR":
evtType = 'MALICIOUS_AFFILIATE_IPADDR'
- if eventName == "INTERNET_NAME":
- evtType = "MALICIOUS_INTERNET_NAME"
- if eventName == 'AFFILIATE_INTERNET_NAME':
- evtType = 'MALICIOUS_AFFILIATE_INTERNET_NAME'
- if eventName == 'CO_HOSTED_SITE':
- evtType = 'MALICIOUS_COHOST'
rec = self.query(addr, "ipr/history")
if rec is not None:
@@ -313,10 +317,6 @@ class sfp_xforce(SpiderFootPlugin):
e = SpiderFootEvent(evtType, entry, self.__name__, event)
self.notifyListeners(e)
- # ipr/malware doesn't support hostnames
- if eventName in ["CO_HOSTED_SITE", "INTERNET_NAME", "AFFILIATE_INTERNET_NAME"]:
- continue
-
rec = self.query(addr, "ipr/malware")
if rec is not None:
rec_malware = rec.get("malware", list())
diff --git a/test/unit/modules/test_sfp_base64.py b/test/unit/modules/test_sfp_base64.py
index e9ca02b6..28f051d1 100644
--- a/test/unit/modules/test_sfp_base64.py
+++ b/test/unit/modules/test_sfp_base64.py
@@ -54,7 +54,7 @@ class TestModuleBase64(unittest.TestCase):
module = sfp_base64()
self.assertIsInstance(module.producedEvents(), list)
- def test_handleEvent_event_data_containing_base64_string_should_return_event(self):
+ def test_handleEvent_event_data_url_containing_base64_string_should_return_event(self):
sf = SpiderFoot(self.default_options)
module = sfp_base64()
@@ -77,7 +77,7 @@ class TestModuleBase64(unittest.TestCase):
module.notifyListeners = new_notifyListeners.__get__(module, sfp_base64)
event_type = 'ROOT'
- event_data = 'example data U3BpZGVyRm9vdA== example data'
+ event_data = 'https://someurl/?somevar=example%20data%20U3BpZGVyRm9vdA%3d%3d%20example%20data'
event_module = ''
source_event = ''