SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.
Python attacksurfacecticybersecurityfootprintinghackinginformation-gatheringinformation-securityinfosecintelligence-gatheringosintosint-frameworkosint-reconnaissanceosint-toolpentestingpythonreconsecurity-toolsthreat-intelligencethreatintel
Threaded S3 bucket checks, must faster now.
Steve Micallefe3e973358ae7
diff --git a/modules/sfp_s3bucket.py b/modules/sfp_s3bucket.pyindex f7c56b55..08ebb0c9 100644--- a/modules/sfp_s3bucket.py+++ b/modules/sfp_s3bucket.py@@ -11,10 +11,10 @@# Licence: GPL# --------------------------------------------------------------------------------#+import threading+import timefrom sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent-class sfp_s3bucket(SpiderFootPlugin):"""S3 Bucket Finder:Footprint,Passive:Crawling and Scanning::Search for potential S3 buckets associated with the target."""@@ -22,20 +22,23 @@ class sfp_s3bucket(SpiderFootPlugin):# Default optionsopts = {"endpoints": "s3.amazonaws.com,s3-external-1.amazonaws.com,s3-us-west-1.amazonaws.com,s3-us-west-2.amazonaws.com,s3.ap-south-1.amazonaws.com,s3-ap-south-1.amazonaws.com,s3.ap-northeast-2.amazonaws.com,s3-ap-northeast-2.amazonaws.com,s3-ap-southeast-1.amazonaws.com,s3-ap-southeast-2.amazonaws.com,s3-ap-northeast-1.amazonaws.com,s3.eu-central-1.amazonaws.com,s3-eu-central-1.amazonaws.com,s3-eu-west-1.amazonaws.com,s3-sa-east-1.amazonaws.com",- "suffixes": "test,dev,web,beta,bucket,-test,-dev,-web,-beta,-bucket"+ "suffixes": "test,dev,web,beta,bucket,-test,-dev,-web,-beta,-bucket",+ "maxthreads": 10}# Option descriptionsoptdescs = {"endpoints": "Different S3 endpoints to check where buckets may exist, as per http://docs.aws.amazon.com/general/latest/gr/rande.html#s3_region",- "suffixes": "List of suffixes to append to domains tried as bucket names"+ "suffixes": "List of suffixes to append to domains tried as bucket names",+ "maxthreads": "Maximum number of threads to spawn for simultaneously checking buckets."}results = list()+ s3results = dict()def setup(self, sfc, userOpts=dict()):self.sf = sfc-+ self.s3results = dict()self.results = list()for opt in userOpts.keys():@@ -51,6 +54,69 @@ class sfp_s3bucket(SpiderFootPlugin):def producedEvents(self):return ["AMAZON_S3_BUCKET"]+ def checkSite(self, url):+ res = self.sf.fetchUrl(url, timeout=10, useragent="SpiderFoot")++ if res['code'] not in [ "301", "302", "200" ] and \+ (res['content'] is None or "NoSuchBucket" in res['content']):+ self.sf.debug("Not a valid bucket: " + url)+ else:+ self.s3results[url] = True++ def threadSites(self, siteList):+ ret = list()+ self.s3results = dict()+ running = True+ i = 0+ t = []++ for site in siteList:+ if self.checkForStop():+ return None++ self.sf.info("Spawning thread to check bucket: " + site)+ t.append(threading.Thread(name='sfp_s3buckets_' + site,+ target=self.checkSite, args=(site,)))+ t[i].start()+ i += 1++ # Block until all threads are finished+ while running:+ found = False+ for rt in threading.enumerate():+ if rt.name.startswith("sfp_s3buckeets_"):+ found = True++ if not found:+ running = False++ time.sleep(1)++ # Return once the scanning has completed+ return self.s3results++ def batchSites(self, sites):+ i = 0+ res = list()+ siteList = list()++ for site in sites:+ if i >= self.opts['maxthreads']:+ data = self.threadSites(siteList)+ if data == None:+ return res++ for ret in data.keys():+ if data[ret]:+ res.append(ret)+ i = 0+ siteList = list()++ siteList.append(site)+ i += 1++ return res+# Handle events sent to this moduledef handleEvent(self, event):eventName = event.eventType@@ -72,6 +138,7 @@ class sfp_s3bucket(SpiderFootPlugin):return Nonetargets = [ eventData.replace('.', ''), self.sf.domainKeyword(eventData, self.opts['_internettlds']) ]+ urls = list()for t in targets:for e in self.opts['endpoints'].split(','):suffixes = [''] + self.opts['suffixes'].split(',')@@ -81,14 +148,11 @@ class sfp_s3bucket(SpiderFootPlugin):b = t + s + "." + eurl = "https://" + b- res = self.sf.fetchUrl(url, timeout=10, useragent="SpiderFoot")-- if res['code'] not in [ "301", "302", "200" ] and \- (res['content'] is None or "NoSuchBucket" in res['content']):- self.sf.debug("Not a valid bucket: " + url)- continue+ urls.append(url)- evt = SpiderFootEvent("AMAZON_S3_BUCKET", b, self.__name__, event)- self.notifyListeners(evt)+ # Batch the scans+ for b in self.batchSites(urls):+ evt = SpiderFootEvent("AMAZON_S3_BUCKET", b, self.__name__, event)+ self.notifyListeners(evt)# End of sfp_s3bucket class