SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.
Python attacksurfacecticybersecurityfootprintinghackinginformation-gatheringinformation-securityinfosecintelligence-gatheringosintosint-frameworkosint-reconnaissanceosint-toolpentestingpythonreconsecurity-toolsthreat-intelligencethreatintel
refactor: update bing_iterate to contain the bing api call
Amr Awad864f726dcf18
diff --git a/modules/sfp_bingsearch.py b/modules/sfp_bingsearch.pyindex c090060e..b99843ad 100644--- a/modules/sfp_bingsearch.py+++ b/modules/sfp_bingsearch.py@@ -9,7 +9,6 @@# Copyright: (c) Steve Micallef 2013# Licence: GPL# --------------------------------------------------------------------------------import requestsfrom sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent@@ -52,32 +51,20 @@ class sfp_bingsearch(SpiderFootPlugin):self.results.append(eventData)# Sites hosted on the domain- endpoint = "https://api.cognitive.microsoft.com/bing/v7.0/search"- # ToDO: figure out how to add credentials in config:- key = ""- response = requests.get(- endpoint,- timeout=self.opts["_fetchtimeout"],- headers={- "Ocp-Apim-Subscription-Key": key,- "User-Agent": self.opts["_useragent"],- },- params={- "q": "site:" + eventData,- "responseFilter": "Webpages",++ results = self.sf.bingIterate(+ searchString="site:" + eventData,+ opts={+ "timeout": self.opts["_fetchtimeout"],+ "useragent": self.opts["_useragent"],"count": self.opts["pages"],},)- if response.status_code != 200:- message = "Failed to talk to Bing API, status code: %s, response: %s".format(- response.status_code, response.text- )- self.error(message)-- links_in_page = [- result["url"] for result in response.json()["webPages"]["value"]- ]- new_links = list(set(links_in_page) - set(self.results))+ if results is None:+ # Failed to talk to bing api+ return None+ urls = results["urls"]+ new_links = list(set(urls) - set(self.results))# Add new links to resultsself.results.extend(new_links)@@ -93,7 +80,7 @@ class sfp_bingsearch(SpiderFootPlugin):if internal_links:# Submit the bing results for analysis- bingsearch_url = response.json()["webPages"]["webSearchUrl"]+ bingsearch_url = results["webSearchUrl"]response = self.sf.fetchUrl(bingsearch_url,timeout=self.opts["_fetchtimeout"],diff --git a/modules/sfp_bingsharedip.py b/modules/sfp_bingsharedip.pyindex 27e98f7a..f3679764 100644--- a/modules/sfp_bingsharedip.py+++ b/modules/sfp_bingsharedip.py@@ -12,7 +12,6 @@import reimport socket-import requestsfrom netaddr import IPNetworkfrom sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent@@ -109,30 +108,18 @@ class sfp_bingsharedip(SpiderFootPlugin):if self.checkForStop():return None- endpoint = "https://api.cognitive.microsoft.com/bing/v7.0/search"- # ToDO: figure out how to add credentials in config:- key = ""- response = requests.get(- endpoint,- timeout=self.opts["_fetchtimeout"],- headers={- "Ocp-Apim-Subscription-Key": key,- "User-Agent": self.opts["_useragent"],- },- params={- "q": "ip:" + ip,- "responseFilter": "Webpages",+ results = self.sf.bingIterate(+ searchString="ip:" + ip,+ opts={+ "timeout": self.opts["_fetchtimeout"],+ "useragent": self.opts["_useragent"],"count": self.opts["pages"],},)-- if response.status_code != 200:- message = "Failed to talk to Bing API, status code: %s, response: %s".format(- response.status_code, response.text- )- self.error(message)-- urls = [result["url"] for result in response.json()["webPages"]["value"]]+ if results is None:+ # Failed to talk to bing api+ return None+ urls = results["urls"]for url in urls:self.sf.info("Found something on same IP: " + url)@@ -168,7 +155,7 @@ class sfp_bingsharedip(SpiderFootPlugin):if urls:# Submit the bing results for analysis- bingsearch_url = response.json()["webPages"]["webSearchUrl"]+ bingsearch_url = results["webSearchUrl"]response = self.sf.fetchUrl(bingsearch_url,timeout=self.opts["_fetchtimeout"],diff --git a/sflib.py b/sflib.pyindex 045fa643..5de2febf 100644--- a/sflib.py+++ b/sflib.py@@ -37,8 +37,10 @@ from copy import deepcopy, copy# For hiding the SSL warnings coming from the requests libimport urllib3+urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)+class SpiderFoot:dbh = NoneGUID = None@@ -75,7 +77,7 @@ class SpiderFoot:try:self.info("Re-circuiting TOR...")- with Controller.from_port(address=self.opts['_socks2addr'],+ with Controller.from_port(address=self.opts['_socks2addr'],port=self.opts['_torctlport']) as controller:controller.authenticate()controller.signal(Signal.NEWNYM)@@ -176,7 +178,7 @@ class SpiderFoot:# Convert supplied raw data into GEXF format (e.g. for Gephi)# GEXF produced by PyGEXF doesn't work with SigmaJS because- # SJS needs coordinates for each node.+ # SJS needs coordinates for each node.# flt is a list of event types to include, if not set everything is# included.def buildGraphGexf(self, root, title, data, flt=[]):@@ -199,7 +201,7 @@ class SpiderFoot:if dst in root:col = ["255", "0", "0"]graph.addNode(str(ncounter), unicode(dst, errors="replace"),- r=col[0], g=col[1], b=col[2])+ r=col[0], g=col[1], b=col[2])nodelist[dst] = ncounterif src not in nodelist:@@ -238,7 +240,7 @@ class SpiderFoot:ncounter = ncounter + 1if dst in root:col = "#f00"- ret['nodes'].append({'id': str(ncounter),+ ret['nodes'].append({'id': str(ncounter),'label': unicode(dst, errors="replace"),'x': random.randint(1,1000),'y': random.randint(1,1000),@@ -251,7 +253,7 @@ class SpiderFoot:if src in root:col = "#f00"ncounter = ncounter + 1- ret['nodes'].append({'id': str(ncounter),+ ret['nodes'].append({'id': str(ncounter),'label': unicode(src, errors="replace"),'x': random.randint(1,1000),'y': random.randint(1,1000),@@ -261,8 +263,8 @@ class SpiderFoot:nodelist[src] = ncounterecounter = ecounter + 1- ret['edges'].append({'id': str(ecounter),- 'source': str(nodelist[src]),+ ret['edges'].append({'id': str(ecounter),+ 'source': str(nodelist[src]),'target': str(nodelist[dst])})@@ -606,7 +608,7 @@ class SpiderFoot:for mod in self.opts['__modules__'].keys():if self.opts['__modules__'][mod]['consumes'] is None:continue-+if "*" in self.opts['__modules__'][mod]['consumes'] and mod not in modlist:modlist.append(mod)@@ -772,7 +774,7 @@ class SpiderFoot:def sanitiseInput(self, cmd):chars = ['a','b','c','d','e','f','g','h','i','j','k','l','m','n','o','p','q','r','s','t','u','v','w','x','y','z',- '0','1','2','3','4','5','6','7','8','9','-','.']+ '0','1','2','3','4','5','6','7','8','9','-','.']for c in cmd:if c.lower() not in chars:return False@@ -1061,7 +1063,7 @@ class SpiderFoot:# Fetch a URL, return the response objectdef fetchUrl(self, url, fatal=False, cookies=None, timeout=30,- useragent="SpiderFoot", headers=None, noLog=False,+ useragent="SpiderFoot", headers=None, noLog=False,postData=None, dontMangle=False, sizeLimit=None,headOnly=False, verify=False):result = {@@ -1242,7 +1244,7 @@ class SpiderFoot:if attempts == 3:return None-+returnResults[seedUrl] = firstPage['content']pat = re.compile("(\/search\S+start=\d+.[^\'\"]*sa=N)", re.IGNORECASE)matches = re.findall(pat, firstPage['content'])@@ -1299,106 +1301,42 @@ class SpiderFoot:return returnResults- # Scrape Bing for content, starting at startUrl and iterating through- # results based on options supplied. Will return a dictionary of all pages- # fetched and their contents {page => content}.+ # Request search results from bing api. Will return a dict:+ # {+ # "urls": a list of urls that match the query string,+ # "webSearchUrl": url for bing results page,+ # }# Options accepted:- # limit: number of search result pages before returning, default is 10- # nopause: don't randomly pause between fetches+ # count: number of search results to request from the api# useragent: User-Agent string to use- # timeout: Fetch timeout+ # timeout: api call timeoutdef bingIterate(self, searchString, opts=dict()):- limit = 10- fetches = 0- returnResults = dict()-- if 'limit' in opts:- limit = opts['limit']-- # We attempt to make the URL look as authentically human as possible- seedUrl = u"http://www.bing.com/search?q={0}".format(searchString) + \- u"&pc=MOZI"-- attempts = 0- failed = False- while attempts < 3:- firstPage = self.fetchUrl(seedUrl, timeout=opts['timeout'],- useragent=opts['useragent'])- if firstPage['code'] == "400":- self.error("Bing doesn't like us right now..", False)- failed = True-- if firstPage['content'] is None:- self.error("Failed to fetch content from Bing.", False)- failed = True- else:- if "/challengepic?" in firstPage['content']:- self.error("Bing returned a CAPTCHA.", False)- failed = True-- if failed:- self.refreshTorIdent()- attempts += 1- failed = False- else:- break-- if attempts == 3:+ endpoint = "https://api.cognitive.microsoft.com/bing/v7.0/search"+ # ToDO: figure out how to add credentials in config:+ key = ""+ response = requests.get(+ endpoint,+ timeout=opts["timeout"],+ headers={"Ocp-Apim-Subscription-Key": key, "User-Agent": opts["useragent"]},+ params={+ "q": searchString,+ "responseFilter": "Webpages",+ "count": opts["count"],+ },+ )+ if response.status_code != 200:+ message = "Failed to talk to Bing API, status code: %s, response: %s".format(+ response.status_code, response.text+ )+ self.error(message)return None- returnResults[seedUrl] = firstPage['content']- pat = re.compile("(\/search\S+first=\d+.[^\'\"]*FORM=\S+)", re.IGNORECASE)- matches = re.findall(pat, firstPage['content'])- while matches > 0 and fetches < limit:- nextUrl = None- fetches += 1- for match in matches:- # Bing moves in increments of 10- if "first=" + str((fetches * 10) + 1) in match:- nextUrl = match.replace("&", "&").replace("%3a", ":")-- if nextUrl is None:- self.debug("Nothing left to scan for in Bing results.")- return returnResults- self.info("Next Bing URL: " + nextUrl)-- # Wait for a random number of seconds between fetches- if 'nopause' not in opts:- pauseSecs = random.randint(4, 15)- self.info("Pausing for " + str(pauseSecs))- time.sleep(pauseSecs)-- attempts = 0- while attempts < 3:- nextPage = self.fetchUrl(u'https://www.bing.com' + nextUrl,- timeout=opts['timeout'], useragent=opts['useragent'])- if nextPage['code'] == "400":- self.error("Bing doesn't like us any more..", False)- failed = True-- if nextPage['content'] is None:- self.error("Failed to fetch subsequent content from Bing.", False)- failed = True- else:- if "/challengepic?" in firstPage['content']:- self.error("Bing returned a CAPTCHA.", False)- failed = True-- if failed:- self.refreshTorIdent()- attempts += 1- failed = False- else:- break-- if attempts == 3:- return returnResults-- returnResults[nextUrl] = nextPage['content']- pat = re.compile("(\/search\S+first=\d+.[^\'\"]*)", re.IGNORECASE)- matches = re.findall(pat, nextPage['content'])+ results = {+ "urls": [result["url"] for result in response.json()["webPages"]["value"]],+ "webSearchUrl": response.json()["webPages"]["webSearchUrl"],+ }+ return results- return returnResults# SpiderFoot plug-in module base class#@@ -1559,7 +1497,7 @@ class SpiderFootPlugin(object):except BaseException as e:f = open("sferror.log", "a")f.write("Module (" + listener.__module__ + ") encountered an error: " + str(e) + "\n")-+exc_type, exc_value, exc_traceback = sys.exc_info()f.write(repr(traceback.format_exception(exc_type, exc_value, exc_traceback)))f.close()@@ -1711,7 +1649,7 @@ class SpiderFootTarget(object):# 2.1if value == name:return True- # 2.2+ # 2.2if includeParents and name.endswith("." + value):return True# 2.3