SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.
Python attacksurfacecticybersecurityfootprintinghackinginformation-gatheringinformation-securityinfosecintelligence-gatheringosintosint-frameworkosint-reconnaissanceosint-toolpentestingpythonreconsecurity-toolsthreat-intelligencethreatintel
Two new modules - identify Google object storage buckets and hashes.
smicalleffa78c0faa1c7
diff --git a/modules/sfp_googleobjectstorage.py b/modules/sfp_googleobjectstorage.pynew file mode 100644index 00000000..2c2d9f2a--- /dev/null+++ b/modules/sfp_googleobjectstorage.py@@ -0,0 +1,170 @@+# -*- coding: utf-8 -*-+# -------------------------------------------------------------------------------+# Name: sfp_googleobjectstorage+# Purpose: SpiderFoot plug-in for identifying potential Google Object Storage+# buckets related to the target.+#+# Author: Steve Micallef <[email protected]>+#+# Created: 24/01/2020+# Copyright: (c) Steve Micallef 2020+# Licence: GPL+# -------------------------------------------------------------------------------++import threading+import time+from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent++class sfp_googleobjectstorage(SpiderFootPlugin):+ """Google Object Storage Finder:Footprint,Passive:Crawling and Scanning::Search for potential Google Object Storage buckets associated with the target and attempt to list their contents."""+++ # Default options+ opts = {+ "suffixes": "test,dev,web,beta,bucket,space,files,content,data,prod,staging,production,stage,app,media,development,-test,-dev,-web,-beta,-bucket,-space,-files,-content,-data,-prod,-staging,-production,-stage,-app,-media,-development",+ "_maxthreads": 20+ }++ # Option descriptions+ optdescs = {+ "suffixes": "List of suffixes to append to domains tried as bucket names"+ }++ results = None+ gosresults = dict()+ lock = None++ def setup(self, sfc, userOpts=dict()):+ self.sf = sfc+ self.gosresults = dict()+ self.results = self.tempStorage()+ self.lock = threading.Lock()++ for opt in list(userOpts.keys()):+ self.opts[opt] = userOpts[opt]++ # What events is this module interested in for input+ def watchedEvents(self):+ return ["DOMAIN_NAME", "LINKED_URL_EXTERNAL"]++ # What events this module produces+ # This is to support the end user in selecting modules based on events+ # produced.+ def producedEvents(self):+ return ["CLOUD_STORAGE_BUCKET", "CLOUD_STORAGE_BUCKET_OPEN"]++ def checkSite(self, url):+ res = self.sf.fetchUrl(url, timeout=10, useragent="SpiderFoot", noLog=True)++ if res['code'] not in [ "301", "302", "200" ] and \+ (res['content'] is None or "NoSuchBucket" in res['content']):+ self.sf.debug("Not a valid bucket: " + url)+ else:+ if "ListBucketResult" in res['content']:+ with self.lock:+ self.gosresults[url] = res['content'].count("<Key>")+ else:+ with self.lock:+ self.gosresults[url] = 0++ def threadSites(self, siteList):+ ret = list()+ self.gosresults = dict()+ running = True+ i = 0+ t = []++ for site in siteList:+ if self.checkForStop():+ return None++ self.sf.info("Spawning thread to check bucket: " + site)+ t.append(threading.Thread(name='thread_sfp_googleobjectstorage_' + site,+ target=self.checkSite, args=(site,)))+ t[i].start()+ i += 1++ # Block until all threads are finished+ while running:+ found = False+ for rt in threading.enumerate():+ if rt.name.startswith("thread_sfp_googleobjectstorage_"):+ found = True++ if not found:+ running = False++ time.sleep(0.25)++ # Return once the scanning has completed+ return self.gosresults++ def batchSites(self, sites):+ i = 0+ res = list()+ siteList = list()++ for site in sites:+ if i >= self.opts['_maxthreads']:+ data = self.threadSites(siteList)+ if data == None:+ return res++ for ret in list(data.keys()):+ if data[ret]:+ # bucket:filecount+ res.append(ret + ":" + str(data[ret]))+ i = 0+ siteList = list()++ siteList.append(site)+ i += 1++ return res++ # Handle events sent to this module+ def handleEvent(self, event):+ eventName = event.eventType+ srcModuleName = event.module+ eventData = event.data++ if eventData in self.results:+ return None+ else:+ self.results[eventData] = True++ self.sf.debug("Received event, " + eventName + ", from " + srcModuleName)++ if eventName == "LINKED_URL_EXTERNAL":+ if ".storage.googleapis.com" in eventData:+ b = self.sf.urlFQDN(eventData)+ evt = SpiderFootEvent("CLOUD_STORAGE_BUCKET", b, self.__name__, event)+ self.notifyListeners(evt)+ return None++ targets = [ eventData.replace('.', ''), self.sf.domainKeyword(eventData, self.opts['_internettlds']) ]+ urls = list()+ for t in targets:+ suffixes = [''] + self.opts['suffixes'].split(',')+ for s in suffixes:+ if self.checkForStop():+ return None++ b = t + s + ".storage.googleapis.com"+ url = "https://" + b+ urls.append(url)++ # Batch the scans+ ret = self.batchSites(urls)+ for b in ret:+ bucket = b.split(":")+ evt = SpiderFootEvent("CLOUD_STORAGE_BUCKET", bucket[0] + ":" + bucket[1], self.__name__, event)+ self.notifyListeners(evt)+ if bucket[2] != "0":+ bucketname = bucket[1].replace("//", "")+ evt = SpiderFootEvent("CLOUD_STORAGE_BUCKET_OPEN", bucketname + ": " + bucket[2] + " files found.",+ self.__name__, evt)+ self.notifyListeners(evt)+++# End of sfp_googleobjectstorage classdiff --git a/modules/sfp_hashes.py b/modules/sfp_hashes.pynew file mode 100644index 00000000..d1809844--- /dev/null+++ b/modules/sfp_hashes.py@@ -0,0 +1,69 @@+# -*- coding: utf-8 -*-+# -------------------------------------------------------------------------------+# Name: sfp_hashes+# Purpose: SpiderFoot plug-in for scanning retreived content by other+# modules (such as sfp_spider) and identifying hashes+#+# Author: Steve Micallef <[email protected]>+#+# Created: 24/01/2020+# Copyright: (c) Steve Micallef 2020+# Licence: GPL+# -------------------------------------------------------------------------------++import re+from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent++class sfp_hashes(SpiderFootPlugin):+ """Hashes:Footprint,Investigate,Passive:Content Analysis::Identify MD5 and SHA hashes in web content, files and more."""++ # Default options+ opts = {+ # options specific to this module+ }++ # Option descriptions+ optdescs = {+ }++ def setup(self, sfc, userOpts=dict()):+ self.sf = sfc++ for opt in userOpts.keys():+ self.opts[opt] = userOpts[opt]++ # What events is this module interested in for input+ def watchedEvents(self):+ return ["TARGET_WEB_CONTENT", "BASE64_DATA",+ "LEAKSITE_CONTENT", "RAW_DNS_RECORDS",+ "RAW_FILE_META_DATA"]++ # What events this module produces+ # This is to support the end user in selecting modules based on events+ # produced.+ def producedEvents(self):+ return ["HASH"]++ # Handle events sent to this module+ def handleEvent(self, event):+ eventName = event.eventType+ srcModuleName = event.module+ eventData = event.data++ self.sf.debug("Received event, " + eventName + ", from " + srcModuleName)++ hashes = self.sf.parseHashes(eventData)+ myres = list()+ for hashtup in hashes:+ hashalgo, hashval = hashtup++ evt = SpiderFootEvent("HASH", "[" + hashalgo + "] " + hashval, self.__name__, event)+ if event.moduleDataSource:+ evt.moduleDataSource = event.moduleDataSource+ else:+ evt.moduleDataSource = "Unknown"+ self.notifyListeners(evt)++ return None++# End of sfp_hashes class