SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.
Python attacksurfacecticybersecurityfootprintinghackinginformation-gatheringinformation-securityinfosecintelligence-gatheringosintosint-frameworkosint-reconnaissanceosint-toolpentestingpythonreconsecurity-toolsthreat-intelligencethreatintel
Implement paging in dehashed
Krishnasis Mandal55bb4f6fc524
diff --git a/modules/sfp_dehashed.py b/modules/sfp_dehashed.pyindex 0b79c3d7..15ce38ab 100644--- a/modules/sfp_dehashed.py+++ b/modules/sfp_dehashed.py@@ -12,6 +12,8 @@# -------------------------------------------------------------------------------from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+import base64+import jsonclass sfp_dehashed(SpiderFootPlugin):@@ -50,7 +52,7 @@ class sfp_dehashed(SpiderFootPlugin):# What events is this module interested in for input# For a list of all events, check sfdb.py.def watchedEvents(self):- return ["EMAILADDR"]+ return ['EMAILADDR']# What events this module producesdef producedEvents(self):@@ -59,24 +61,30 @@ class sfp_dehashed(SpiderFootPlugin):# When querying third parties, it's best to have a dedicated function# to do so and avoid putting it in handleEvent()- def query(self, qry):+ def query(self, qry, currentPage):+ b64_auth = base64.b64encode((self.opts['email'] + ":" + self.opts['api_key']).encode("utf-8"))headers = {'Accept' : 'application/json',- 'Authorization': "Basic " + base64.b64encode(self.opts['email'] + ":" + self.opts['api_key'])+ 'Authorization': "Basic " + b64_auth.decode("utf-8")}- res = self.sf.fetchUrl("https://api.dehashed.com/search?query=\"" + qry + "\"",+ self.sf.debug(str(headers))+ res = self.sf.fetchUrl("https://api.dehashed.com/search?query=\"" + qry + "\"" + "&page=" + str(currentPage),headers=headers,timeout=15,useragent=self.opts['_useragent'])-- if res['content'] is None:- self.sf.info("No Dehashed info found for " + qry)++ self.sf.debug(str(res))+ self.sf.debug("CONTENT : ::::::::" + str(res.get('content')))+ if res.get('code') == '401':+ return None+ if res.get('content') is None:+ self.sf.debug("No Dehashed info found for " + qry)return None# Always always always process external data with try/except since we cannot# trust the data is as intended.try:- info = json.loads(res['content'])+ info = json.loads(res.get('content'))except Exception as e:self.sf.error("Error processing JSON response from Dehashed.", False)return None@@ -85,6 +93,7 @@ class sfp_dehashed(SpiderFootPlugin):# Handle events sent to this moduledef handleEvent(self, event):+ self.sf.debug("Testing Dehashed . ")eventName = event.eventTypesrcModuleName = event.moduleeventData = event.data@@ -112,15 +121,28 @@ class sfp_dehashed(SpiderFootPlugin):self.results[eventData] = True- entries = list()# To implement : Number of pages# Fetch Dehashed data for incoming data (email)- jsonData = self.query(eventData)-- if jsonData is None:- return None+ self.sf.debug("Starting querying from DeHashed")+ currentPage = 1+ entries = list()+ self.sf.debug("Current Page : " + str(currentPage))+ self.sf.debug("Max Pages : " + self.opts['max_pages'])+ while currentPage <= int(self.opts['max_pages']):+ self.sf.debug("Looking for page : " + str(currentPage))+ jsonData_temp = self.query(eventData, currentPage)+ if jsonData_temp is None:+ self.sf.error("No Data receieved")+ break+ else:+ self.sf.debug("Received : " + str(entries))+ entries.append(jsonData_temp.get('entries'))+ # Dehashed returns 5000 results for each query+ if len(entries) >= 5000 * currentPage:+ currentPage += 1+ else:+ break- entries = jsonData.get('entries')if entries is None:return None