Skip to content

SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.

S3 bucket listing and new Digital Ocean Spaces module.

Steve Micallef3f2bee6622c0
diff --git a/modules/sfp_digitaloceanspace.py b/modules/sfp_digitaloceanspace.py
new file mode 100644
index 00000000..50994e45
--- /dev/null
+++ b/modules/sfp_digitaloceanspace.py
@@ -0,0 +1,168 @@
+# -*- coding: utf-8 -*-
+# -------------------------------------------------------------------------------
+# Name: sfp_digitaloceanspace
+# Purpose: SpiderFoot plug-in for identifying potential Digital Ocean spaces
+# related to the target.
+#
+# Author: Steve Micallef <[email protected]>
+#
+# Created: 16/06/2018
+# Copyright: (c) Steve Micallef 2018
+# Licence: GPL
+# -------------------------------------------------------------------------------
+
+import threading
+import time
+from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+
+class sfp_digitaloceanspace(SpiderFootPlugin):
+ """Digital Ocean Space Finder:Footprint,Passive:Crawling and Scanning::Search for potential Digital Ocean Spaces associated with the target and attempt to list their contents."""
+
+
+ # Default options
+ opts = {
+ "endpoints": "nyc3.digitaloceanspaces.com,sgp1.digitaloceanspaces.com,ams3.digitaloceanspaces.com",
+ "suffixes": "test,dev,web,beta,bucket,space,files,content,data,-test,-dev,-web,-beta,-bucket,-space,-files,-content,-data",
+ "_maxthreads": 20
+ }
+
+ # Option descriptions
+ optdescs = {
+ "endpoints": "Different Digital Ocean locations to check where spaces may exist.",
+ "suffixes": "List of suffixes to append to domains tried as space names"
+ }
+
+ results = list()
+ s3results = dict()
+
+ def setup(self, sfc, userOpts=dict()):
+ self.sf = sfc
+ self.s3results = dict()
+ self.results = list()
+
+ for opt in userOpts.keys():
+ self.opts[opt] = userOpts[opt]
+
+ # What events is this module interested in for input
+ def watchedEvents(self):
+ return ["DOMAIN_NAME", "LINKED_URL_EXTERNAL"]
+
+ # What events this module produces
+ # This is to support the end user in selecting modules based on events
+ # produced.
+ def producedEvents(self):
+ return ["CLOUD_STORAGE_BUCKET", "CLOUD_STORAGE_BUCKET_OPEN"]
+
+ def checkSite(self, url):
+ res = self.sf.fetchUrl(url, timeout=10, useragent="SpiderFoot")
+
+ if res['code'] not in [ "301", "302", "200" ] and \
+ (res['content'] is None or "NoSuchBucket" in res['content']):
+ self.sf.debug("Not a valid bucket: " + url)
+ else:
+ if "ListBucketResult" in res['content']:
+ self.s3results[url] = res['content'].count("<Key>")
+ else:
+ self.s3results[url] = 0
+
+ def threadSites(self, siteList):
+ ret = list()
+ self.s3results = dict()
+ running = True
+ i = 0
+ t = []
+
+ for site in siteList:
+ if self.checkForStop():
+ return None
+
+ self.sf.info("Spawning thread to check bucket: " + site)
+ t.append(threading.Thread(name='sfp_digitaloceanspaces_' + site,
+ target=self.checkSite, args=(site,)))
+ t[i].start()
+ i += 1
+
+ # Block until all threads are finished
+ while running:
+ found = False
+ for rt in threading.enumerate():
+ if rt.name.startswith("sfp_s3buckeets_"):
+ found = True
+
+ if not found:
+ running = False
+
+ time.sleep(0.25)
+
+ # Return once the scanning has completed
+ return self.s3results
+
+ def batchSites(self, sites):
+ i = 0
+ res = list()
+ siteList = list()
+
+ for site in sites:
+ if i >= self.opts['_maxthreads']:
+ data = self.threadSites(siteList)
+ if data == None:
+ return res
+
+ for ret in data.keys():
+ if data[ret]:
+ # bucket:filecount
+ res.append(ret + ":" + str(data[ret]))
+ i = 0
+ siteList = list()
+
+ siteList.append(site)
+ i += 1
+
+ return res
+
+ # Handle events sent to this module
+ def handleEvent(self, event):
+ eventName = event.eventType
+ srcModuleName = event.module
+ eventData = event.data
+
+ if eventData in self.results:
+ return None
+ else:
+ self.results.append(eventData)
+
+ self.sf.debug("Received event, " + eventName + ", from " + srcModuleName)
+
+ if eventName == "LINKED_URL_EXTERNAL":
+ if ".digitaloceanspaces.com" in eventData:
+ b = self.sf.urlFQDN(eventData)
+ evt = SpiderFootEvent("CLOUD_STORAGE_BUCKET", b, self.__name__, event)
+ self.notifyListeners(evt)
+ return None
+
+ targets = [ eventData.replace('.', ''), self.sf.domainKeyword(eventData, self.opts['_internettlds']) ]
+ urls = list()
+ for t in targets:
+ for e in self.opts['endpoints'].split(','):
+ suffixes = [''] + self.opts['suffixes'].split(',')
+ for s in suffixes:
+ if self.checkForStop():
+ return None
+
+ b = t + s + "." + e
+ url = "https://" + b
+ urls.append(url)
+
+ # Batch the scans
+ ret = self.batchSites(urls)
+ for b in ret:
+ bucket = b.split(":")
+ evt = SpiderFootEvent("CLOUD_STORAGE_BUCKET", bucket[0] + ":" + bucket[1], self.__name__, event)
+ self.notifyListeners(evt)
+ if bucket[2] != "0":
+ evt = SpiderFootEvent("CLOUD_STORAGE_BUCKET_OPEN", bucket[2] + " files found.",
+ self.__name__, evt)
+ self.notifyListeners(evt)
+
+
+# End of sfp_digitaloceanspace class
diff --git a/modules/sfp_s3bucket.py b/modules/sfp_s3bucket.py
index 08ebb0c9..488d3bdc 100644
--- a/modules/sfp_s3bucket.py
+++ b/modules/sfp_s3bucket.py
@@ -16,21 +16,20 @@ import time
from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
class sfp_s3bucket(SpiderFootPlugin):
- """S3 Bucket Finder:Footprint,Passive:Crawling and Scanning::Search for potential S3 buckets associated with the target."""
+ """Amazon S3 Bucket Finder:Footprint,Passive:Crawling and Scanning::Search for potential Amazon S3 buckets associated with the target and attempt to list their contents."""
# Default options
opts = {
"endpoints": "s3.amazonaws.com,s3-external-1.amazonaws.com,s3-us-west-1.amazonaws.com,s3-us-west-2.amazonaws.com,s3.ap-south-1.amazonaws.com,s3-ap-south-1.amazonaws.com,s3.ap-northeast-2.amazonaws.com,s3-ap-northeast-2.amazonaws.com,s3-ap-southeast-1.amazonaws.com,s3-ap-southeast-2.amazonaws.com,s3-ap-northeast-1.amazonaws.com,s3.eu-central-1.amazonaws.com,s3-eu-central-1.amazonaws.com,s3-eu-west-1.amazonaws.com,s3-sa-east-1.amazonaws.com",
- "suffixes": "test,dev,web,beta,bucket,-test,-dev,-web,-beta,-bucket",
- "maxthreads": 10
+ "suffixes": "test,dev,web,beta,bucket,space,files,content,data,-test,-dev,-web,-beta,-bucket,-space,-files,-content,-data",
+ "_maxthreads": 20
}
# Option descriptions
optdescs = {
"endpoints": "Different S3 endpoints to check where buckets may exist, as per http://docs.aws.amazon.com/general/latest/gr/rande.html#s3_region",
- "suffixes": "List of suffixes to append to domains tried as bucket names",
- "maxthreads": "Maximum number of threads to spawn for simultaneously checking buckets."
+ "suffixes": "List of suffixes to append to domains tried as bucket names"
}
results = list()
@@ -52,7 +51,7 @@ class sfp_s3bucket(SpiderFootPlugin):
# This is to support the end user in selecting modules based on events
# produced.
def producedEvents(self):
- return ["AMAZON_S3_BUCKET"]
+ return ["CLOUD_STORAGE_BUCKET", "CLOUD_STORAGE_BUCKET_OPEN"]
def checkSite(self, url):
res = self.sf.fetchUrl(url, timeout=10, useragent="SpiderFoot")
@@ -61,7 +60,10 @@ class sfp_s3bucket(SpiderFootPlugin):
(res['content'] is None or "NoSuchBucket" in res['content']):
self.sf.debug("Not a valid bucket: " + url)
else:
- self.s3results[url] = True
+ if "ListBucketResult" in res['content']:
+ self.s3results[url] = res['content'].count("<Key>")
+ else:
+ self.s3results[url] = 0
def threadSites(self, siteList):
ret = list()
@@ -90,7 +92,7 @@ class sfp_s3bucket(SpiderFootPlugin):
if not found:
running = False
- time.sleep(1)
+ time.sleep(0.25)
# Return once the scanning has completed
return self.s3results
@@ -101,14 +103,15 @@ class sfp_s3bucket(SpiderFootPlugin):
siteList = list()
for site in sites:
- if i >= self.opts['maxthreads']:
+ if i >= self.opts['_maxthreads']:
data = self.threadSites(siteList)
if data == None:
return res
for ret in data.keys():
if data[ret]:
- res.append(ret)
+ # bucket:filecount
+ res.append(ret + ":" + str(data[ret]))
i = 0
siteList = list()
@@ -133,7 +136,7 @@ class sfp_s3bucket(SpiderFootPlugin):
if eventName == "LINKED_URL_EXTERNAL":
if ".amazonaws.com" in eventData:
b = self.sf.urlFQDN(eventData)
- evt = SpiderFootEvent("AMAZON_S3_BUCKET", b, self.__name__, event)
+ evt = SpiderFootEvent("CLOUD_STORAGE_BUCKET", b, self.__name__, event)
self.notifyListeners(evt)
return None
@@ -151,8 +154,15 @@ class sfp_s3bucket(SpiderFootPlugin):
urls.append(url)
# Batch the scans
- for b in self.batchSites(urls):
- evt = SpiderFootEvent("AMAZON_S3_BUCKET", b, self.__name__, event)
+ ret = self.batchSites(urls)
+ for b in ret:
+ bucket = b.split(":")
+ evt = SpiderFootEvent("CLOUD_STORAGE_BUCKET", bucket[0] + ":" + bucket[1], self.__name__, event)
self.notifyListeners(evt)
+ if bucket[2] != "0":
+ evt = SpiderFootEvent("CLOUD_STORAGE_BUCKET_OPEN", bucket[2] + " files found.",
+ self.__name__, evt)
+ self.notifyListeners(evt)
+
# End of sfp_s3bucket class
diff --git a/sfdb.py b/sfdb.py
index ea6a98b1..f34b19b0 100644
--- a/sfdb.py
+++ b/sfdb.py
@@ -91,6 +91,7 @@ class SpiderFootDb:
"INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('ACCOUNT_EXTERNAL_OWNED_COMPROMISED', 'Hacked Account on External Site', 0, 'DESCRIPTOR')",
"INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('ACCOUNT_EXTERNAL_USER_SHARED', 'User Account on External Site', 0, 'ENTITY')",
"INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('ACCOUNT_EXTERNAL_USER_SHARED_COMPROMISED', 'Hacked User Account on External Site', 0, 'DESCRIPTOR')",
+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('AFFILIATE_EMAILADDR', 'Affiliate - Email Address', 0, 'ENTITY')",
"INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('AFFILIATE_INTERNET_NAME', 'Affiliate - Internet Name', 0, 'ENTITY')",
"INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('AFFILIATE_IPADDR', 'Affiliate - IP Address', 0, 'ENTITY')",
"INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('AFFILIATE_WEB_CONTENT', 'Affiliate - Web Content', 1, 'DATA')",
@@ -99,7 +100,8 @@ class SpiderFootDb:
"INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('AFFILIATE_DESCRIPTION_CATEGORY', 'Affiliate Description - Category', 0, 'DESCRIPTOR')",
"INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('AFFILIATE_DESCRIPTION_ABSTRACT', 'Affiliate Description - Abstract', 0, 'DESCRIPTOR')",
"INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('APPSTORE_ENTRY', 'App Store Entry', 0, 'ENTITY')",
- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('AMAZON_S3_BUCKET', 'Amazon S3 Bucket', 0, 'ENTITY')",
+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('CLOUD_STORAGE_BUCKET', 'Cloud Storage Bucket', 0, 'ENTITY')",
+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('CLOUD_STORAGE_BUCKET_OPEN', 'Cloud Storage Bucket Open', 0, 'DESCRIPTOR')",
"INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('BASE64_DATA', 'Base64-encoded Data', 1, 'DATA')",
"INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('BITCOIN_ADDRESS', 'Bitcoin Address', 0, 'ENTITY')",
"INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('BITCOIN_BALANCE', 'Bitcoin Balance', 0, 'DESCRIPTOR')",
@@ -140,6 +142,7 @@ class SpiderFootDb:
"INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('INTERESTING_FILE_HISTORIC', 'Historic Interesting File', 0, 'DESCRIPTOR')",
"INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('JUNK_FILE', 'Junk File', 0, 'DESCRIPTOR')",
"INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('INTERNET_NAME', 'Internet Name', 0, 'ENTITY')",
+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('INTERNET_NAME_UNRESOLVED', 'Internet Name - Unresolved', 0, 'ENTITY')",
"INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('IP_ADDRESS', 'IP Address', 0, 'ENTITY')",
"INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('IPV6_ADDRESS', 'IPv6 Address', 0, 'ENTITY')",
"INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('LINKED_URL_INTERNAL', 'Linked URL - Internal', 0, 'SUBENTITY')",