SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.
Python attacksurfacecticybersecurityfootprintinghackinginformation-gatheringinformation-securityinfosecintelligence-gatheringosintosint-frameworkosint-reconnaissanceosint-toolpentestingpythonreconsecurity-toolsthreat-intelligencethreatintel
S3 bucket listing and new Digital Ocean Spaces module.
Steve Micallef3f2bee6622c0
diff --git a/modules/sfp_digitaloceanspace.py b/modules/sfp_digitaloceanspace.pynew file mode 100644index 00000000..50994e45--- /dev/null+++ b/modules/sfp_digitaloceanspace.py@@ -0,0 +1,168 @@+# -*- coding: utf-8 -*-+# -------------------------------------------------------------------------------+# Name: sfp_digitaloceanspace+# Purpose: SpiderFoot plug-in for identifying potential Digital Ocean spaces+# related to the target.+#+# Author: Steve Micallef <[email protected]>+#+# Created: 16/06/2018+# Copyright: (c) Steve Micallef 2018+# Licence: GPL+# -------------------------------------------------------------------------------++import threading+import time+from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent++class sfp_digitaloceanspace(SpiderFootPlugin):+ """Digital Ocean Space Finder:Footprint,Passive:Crawling and Scanning::Search for potential Digital Ocean Spaces associated with the target and attempt to list their contents."""+++ # Default options+ opts = {+ "endpoints": "nyc3.digitaloceanspaces.com,sgp1.digitaloceanspaces.com,ams3.digitaloceanspaces.com",+ "suffixes": "test,dev,web,beta,bucket,space,files,content,data,-test,-dev,-web,-beta,-bucket,-space,-files,-content,-data",+ "_maxthreads": 20+ }++ # Option descriptions+ optdescs = {+ "endpoints": "Different Digital Ocean locations to check where spaces may exist.",+ "suffixes": "List of suffixes to append to domains tried as space names"+ }++ results = list()+ s3results = dict()++ def setup(self, sfc, userOpts=dict()):+ self.sf = sfc+ self.s3results = dict()+ self.results = list()++ for opt in userOpts.keys():+ self.opts[opt] = userOpts[opt]++ # What events is this module interested in for input+ def watchedEvents(self):+ return ["DOMAIN_NAME", "LINKED_URL_EXTERNAL"]++ # What events this module produces+ # This is to support the end user in selecting modules based on events+ # produced.+ def producedEvents(self):+ return ["CLOUD_STORAGE_BUCKET", "CLOUD_STORAGE_BUCKET_OPEN"]++ def checkSite(self, url):+ res = self.sf.fetchUrl(url, timeout=10, useragent="SpiderFoot")++ if res['code'] not in [ "301", "302", "200" ] and \+ (res['content'] is None or "NoSuchBucket" in res['content']):+ self.sf.debug("Not a valid bucket: " + url)+ else:+ if "ListBucketResult" in res['content']:+ self.s3results[url] = res['content'].count("<Key>")+ else:+ self.s3results[url] = 0++ def threadSites(self, siteList):+ ret = list()+ self.s3results = dict()+ running = True+ i = 0+ t = []++ for site in siteList:+ if self.checkForStop():+ return None++ self.sf.info("Spawning thread to check bucket: " + site)+ t.append(threading.Thread(name='sfp_digitaloceanspaces_' + site,+ target=self.checkSite, args=(site,)))+ t[i].start()+ i += 1++ # Block until all threads are finished+ while running:+ found = False+ for rt in threading.enumerate():+ if rt.name.startswith("sfp_s3buckeets_"):+ found = True++ if not found:+ running = False++ time.sleep(0.25)++ # Return once the scanning has completed+ return self.s3results++ def batchSites(self, sites):+ i = 0+ res = list()+ siteList = list()++ for site in sites:+ if i >= self.opts['_maxthreads']:+ data = self.threadSites(siteList)+ if data == None:+ return res++ for ret in data.keys():+ if data[ret]:+ # bucket:filecount+ res.append(ret + ":" + str(data[ret]))+ i = 0+ siteList = list()++ siteList.append(site)+ i += 1++ return res++ # Handle events sent to this module+ def handleEvent(self, event):+ eventName = event.eventType+ srcModuleName = event.module+ eventData = event.data++ if eventData in self.results:+ return None+ else:+ self.results.append(eventData)++ self.sf.debug("Received event, " + eventName + ", from " + srcModuleName)++ if eventName == "LINKED_URL_EXTERNAL":+ if ".digitaloceanspaces.com" in eventData:+ b = self.sf.urlFQDN(eventData)+ evt = SpiderFootEvent("CLOUD_STORAGE_BUCKET", b, self.__name__, event)+ self.notifyListeners(evt)+ return None++ targets = [ eventData.replace('.', ''), self.sf.domainKeyword(eventData, self.opts['_internettlds']) ]+ urls = list()+ for t in targets:+ for e in self.opts['endpoints'].split(','):+ suffixes = [''] + self.opts['suffixes'].split(',')+ for s in suffixes:+ if self.checkForStop():+ return None++ b = t + s + "." + e+ url = "https://" + b+ urls.append(url)++ # Batch the scans+ ret = self.batchSites(urls)+ for b in ret:+ bucket = b.split(":")+ evt = SpiderFootEvent("CLOUD_STORAGE_BUCKET", bucket[0] + ":" + bucket[1], self.__name__, event)+ self.notifyListeners(evt)+ if bucket[2] != "0":+ evt = SpiderFootEvent("CLOUD_STORAGE_BUCKET_OPEN", bucket[2] + " files found.",+ self.__name__, evt)+ self.notifyListeners(evt)+++# End of sfp_digitaloceanspace classdiff --git a/modules/sfp_s3bucket.py b/modules/sfp_s3bucket.pyindex 08ebb0c9..488d3bdc 100644--- a/modules/sfp_s3bucket.py+++ b/modules/sfp_s3bucket.py@@ -16,21 +16,20 @@ import timefrom sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEventclass sfp_s3bucket(SpiderFootPlugin):- """S3 Bucket Finder:Footprint,Passive:Crawling and Scanning::Search for potential S3 buckets associated with the target."""+ """Amazon S3 Bucket Finder:Footprint,Passive:Crawling and Scanning::Search for potential Amazon S3 buckets associated with the target and attempt to list their contents."""# Default optionsopts = {"endpoints": "s3.amazonaws.com,s3-external-1.amazonaws.com,s3-us-west-1.amazonaws.com,s3-us-west-2.amazonaws.com,s3.ap-south-1.amazonaws.com,s3-ap-south-1.amazonaws.com,s3.ap-northeast-2.amazonaws.com,s3-ap-northeast-2.amazonaws.com,s3-ap-southeast-1.amazonaws.com,s3-ap-southeast-2.amazonaws.com,s3-ap-northeast-1.amazonaws.com,s3.eu-central-1.amazonaws.com,s3-eu-central-1.amazonaws.com,s3-eu-west-1.amazonaws.com,s3-sa-east-1.amazonaws.com",- "suffixes": "test,dev,web,beta,bucket,-test,-dev,-web,-beta,-bucket",- "maxthreads": 10+ "suffixes": "test,dev,web,beta,bucket,space,files,content,data,-test,-dev,-web,-beta,-bucket,-space,-files,-content,-data",+ "_maxthreads": 20}# Option descriptionsoptdescs = {"endpoints": "Different S3 endpoints to check where buckets may exist, as per http://docs.aws.amazon.com/general/latest/gr/rande.html#s3_region",- "suffixes": "List of suffixes to append to domains tried as bucket names",- "maxthreads": "Maximum number of threads to spawn for simultaneously checking buckets."+ "suffixes": "List of suffixes to append to domains tried as bucket names"}results = list()@@ -52,7 +51,7 @@ class sfp_s3bucket(SpiderFootPlugin):# This is to support the end user in selecting modules based on events# produced.def producedEvents(self):- return ["AMAZON_S3_BUCKET"]+ return ["CLOUD_STORAGE_BUCKET", "CLOUD_STORAGE_BUCKET_OPEN"]def checkSite(self, url):res = self.sf.fetchUrl(url, timeout=10, useragent="SpiderFoot")@@ -61,7 +60,10 @@ class sfp_s3bucket(SpiderFootPlugin):(res['content'] is None or "NoSuchBucket" in res['content']):self.sf.debug("Not a valid bucket: " + url)else:- self.s3results[url] = True+ if "ListBucketResult" in res['content']:+ self.s3results[url] = res['content'].count("<Key>")+ else:+ self.s3results[url] = 0def threadSites(self, siteList):ret = list()@@ -90,7 +92,7 @@ class sfp_s3bucket(SpiderFootPlugin):if not found:running = False- time.sleep(1)+ time.sleep(0.25)# Return once the scanning has completedreturn self.s3results@@ -101,14 +103,15 @@ class sfp_s3bucket(SpiderFootPlugin):siteList = list()for site in sites:- if i >= self.opts['maxthreads']:+ if i >= self.opts['_maxthreads']:data = self.threadSites(siteList)if data == None:return resfor ret in data.keys():if data[ret]:- res.append(ret)+ # bucket:filecount+ res.append(ret + ":" + str(data[ret]))i = 0siteList = list()@@ -133,7 +136,7 @@ class sfp_s3bucket(SpiderFootPlugin):if eventName == "LINKED_URL_EXTERNAL":if ".amazonaws.com" in eventData:b = self.sf.urlFQDN(eventData)- evt = SpiderFootEvent("AMAZON_S3_BUCKET", b, self.__name__, event)+ evt = SpiderFootEvent("CLOUD_STORAGE_BUCKET", b, self.__name__, event)self.notifyListeners(evt)return None@@ -151,8 +154,15 @@ class sfp_s3bucket(SpiderFootPlugin):urls.append(url)# Batch the scans- for b in self.batchSites(urls):- evt = SpiderFootEvent("AMAZON_S3_BUCKET", b, self.__name__, event)+ ret = self.batchSites(urls)+ for b in ret:+ bucket = b.split(":")+ evt = SpiderFootEvent("CLOUD_STORAGE_BUCKET", bucket[0] + ":" + bucket[1], self.__name__, event)self.notifyListeners(evt)+ if bucket[2] != "0":+ evt = SpiderFootEvent("CLOUD_STORAGE_BUCKET_OPEN", bucket[2] + " files found.",+ self.__name__, evt)+ self.notifyListeners(evt)+# End of sfp_s3bucket classdiff --git a/sfdb.py b/sfdb.pyindex ea6a98b1..f34b19b0 100644--- a/sfdb.py+++ b/sfdb.py@@ -91,6 +91,7 @@ class SpiderFootDb:"INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('ACCOUNT_EXTERNAL_OWNED_COMPROMISED', 'Hacked Account on External Site', 0, 'DESCRIPTOR')","INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('ACCOUNT_EXTERNAL_USER_SHARED', 'User Account on External Site', 0, 'ENTITY')","INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('ACCOUNT_EXTERNAL_USER_SHARED_COMPROMISED', 'Hacked User Account on External Site', 0, 'DESCRIPTOR')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('AFFILIATE_EMAILADDR', 'Affiliate - Email Address', 0, 'ENTITY')","INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('AFFILIATE_INTERNET_NAME', 'Affiliate - Internet Name', 0, 'ENTITY')","INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('AFFILIATE_IPADDR', 'Affiliate - IP Address', 0, 'ENTITY')","INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('AFFILIATE_WEB_CONTENT', 'Affiliate - Web Content', 1, 'DATA')",@@ -99,7 +100,8 @@ class SpiderFootDb:"INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('AFFILIATE_DESCRIPTION_CATEGORY', 'Affiliate Description - Category', 0, 'DESCRIPTOR')","INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('AFFILIATE_DESCRIPTION_ABSTRACT', 'Affiliate Description - Abstract', 0, 'DESCRIPTOR')","INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('APPSTORE_ENTRY', 'App Store Entry', 0, 'ENTITY')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('AMAZON_S3_BUCKET', 'Amazon S3 Bucket', 0, 'ENTITY')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('CLOUD_STORAGE_BUCKET', 'Cloud Storage Bucket', 0, 'ENTITY')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('CLOUD_STORAGE_BUCKET_OPEN', 'Cloud Storage Bucket Open', 0, 'DESCRIPTOR')","INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('BASE64_DATA', 'Base64-encoded Data', 1, 'DATA')","INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('BITCOIN_ADDRESS', 'Bitcoin Address', 0, 'ENTITY')","INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('BITCOIN_BALANCE', 'Bitcoin Balance', 0, 'DESCRIPTOR')",@@ -140,6 +142,7 @@ class SpiderFootDb:"INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('INTERESTING_FILE_HISTORIC', 'Historic Interesting File', 0, 'DESCRIPTOR')","INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('JUNK_FILE', 'Junk File', 0, 'DESCRIPTOR')","INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('INTERNET_NAME', 'Internet Name', 0, 'ENTITY')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('INTERNET_NAME_UNRESOLVED', 'Internet Name - Unresolved', 0, 'ENTITY')","INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('IP_ADDRESS', 'IP Address', 0, 'ENTITY')","INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('IPV6_ADDRESS', 'IPv6 Address', 0, 'ENTITY')","INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('LINKED_URL_INTERNAL', 'Linked URL - Internal', 0, 'SUBENTITY')",