Skip to content

SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.

refactor: update bing_iterate to contain the bing api call

Amr Awad864f726dcf18
diff --git a/modules/sfp_bingsearch.py b/modules/sfp_bingsearch.py
index c090060e..b99843ad 100644
--- a/modules/sfp_bingsearch.py
+++ b/modules/sfp_bingsearch.py
@@ -9,7 +9,6 @@
# Copyright: (c) Steve Micallef 2013
# Licence: GPL
# -------------------------------------------------------------------------------
-import requests
from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
@@ -52,32 +51,20 @@ class sfp_bingsearch(SpiderFootPlugin):
self.results.append(eventData)
# Sites hosted on the domain
- endpoint = "https://api.cognitive.microsoft.com/bing/v7.0/search"
- # ToDO: figure out how to add credentials in config:
- key = ""
- response = requests.get(
- endpoint,
- timeout=self.opts["_fetchtimeout"],
- headers={
- "Ocp-Apim-Subscription-Key": key,
- "User-Agent": self.opts["_useragent"],
- },
- params={
- "q": "site:" + eventData,
- "responseFilter": "Webpages",
+
+ results = self.sf.bingIterate(
+ searchString="site:" + eventData,
+ opts={
+ "timeout": self.opts["_fetchtimeout"],
+ "useragent": self.opts["_useragent"],
"count": self.opts["pages"],
},
)
- if response.status_code != 200:
- message = "Failed to talk to Bing API, status code: %s, response: %s".format(
- response.status_code, response.text
- )
- self.error(message)
-
- links_in_page = [
- result["url"] for result in response.json()["webPages"]["value"]
- ]
- new_links = list(set(links_in_page) - set(self.results))
+ if results is None:
+ # Failed to talk to bing api
+ return None
+ urls = results["urls"]
+ new_links = list(set(urls) - set(self.results))
# Add new links to results
self.results.extend(new_links)
@@ -93,7 +80,7 @@ class sfp_bingsearch(SpiderFootPlugin):
if internal_links:
# Submit the bing results for analysis
- bingsearch_url = response.json()["webPages"]["webSearchUrl"]
+ bingsearch_url = results["webSearchUrl"]
response = self.sf.fetchUrl(
bingsearch_url,
timeout=self.opts["_fetchtimeout"],
diff --git a/modules/sfp_bingsharedip.py b/modules/sfp_bingsharedip.py
index 27e98f7a..f3679764 100644
--- a/modules/sfp_bingsharedip.py
+++ b/modules/sfp_bingsharedip.py
@@ -12,7 +12,6 @@
import re
import socket
-import requests
from netaddr import IPNetwork
from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
@@ -109,30 +108,18 @@ class sfp_bingsharedip(SpiderFootPlugin):
if self.checkForStop():
return None
- endpoint = "https://api.cognitive.microsoft.com/bing/v7.0/search"
- # ToDO: figure out how to add credentials in config:
- key = ""
- response = requests.get(
- endpoint,
- timeout=self.opts["_fetchtimeout"],
- headers={
- "Ocp-Apim-Subscription-Key": key,
- "User-Agent": self.opts["_useragent"],
- },
- params={
- "q": "ip:" + ip,
- "responseFilter": "Webpages",
+ results = self.sf.bingIterate(
+ searchString="ip:" + ip,
+ opts={
+ "timeout": self.opts["_fetchtimeout"],
+ "useragent": self.opts["_useragent"],
"count": self.opts["pages"],
},
)
-
- if response.status_code != 200:
- message = "Failed to talk to Bing API, status code: %s, response: %s".format(
- response.status_code, response.text
- )
- self.error(message)
-
- urls = [result["url"] for result in response.json()["webPages"]["value"]]
+ if results is None:
+ # Failed to talk to bing api
+ return None
+ urls = results["urls"]
for url in urls:
self.sf.info("Found something on same IP: " + url)
@@ -168,7 +155,7 @@ class sfp_bingsharedip(SpiderFootPlugin):
if urls:
# Submit the bing results for analysis
- bingsearch_url = response.json()["webPages"]["webSearchUrl"]
+ bingsearch_url = results["webSearchUrl"]
response = self.sf.fetchUrl(
bingsearch_url,
timeout=self.opts["_fetchtimeout"],
diff --git a/sflib.py b/sflib.py
index 045fa643..5de2febf 100644
--- a/sflib.py
+++ b/sflib.py
@@ -37,8 +37,10 @@ from copy import deepcopy, copy
# For hiding the SSL warnings coming from the requests lib
import urllib3
+
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
+
class SpiderFoot:
dbh = None
GUID = None
@@ -75,7 +77,7 @@ class SpiderFoot:
try:
self.info("Re-circuiting TOR...")
- with Controller.from_port(address=self.opts['_socks2addr'],
+ with Controller.from_port(address=self.opts['_socks2addr'],
port=self.opts['_torctlport']) as controller:
controller.authenticate()
controller.signal(Signal.NEWNYM)
@@ -176,7 +178,7 @@ class SpiderFoot:
# Convert supplied raw data into GEXF format (e.g. for Gephi)
# GEXF produced by PyGEXF doesn't work with SigmaJS because
- # SJS needs coordinates for each node.
+ # SJS needs coordinates for each node.
# flt is a list of event types to include, if not set everything is
# included.
def buildGraphGexf(self, root, title, data, flt=[]):
@@ -199,7 +201,7 @@ class SpiderFoot:
if dst in root:
col = ["255", "0", "0"]
graph.addNode(str(ncounter), unicode(dst, errors="replace"),
- r=col[0], g=col[1], b=col[2])
+ r=col[0], g=col[1], b=col[2])
nodelist[dst] = ncounter
if src not in nodelist:
@@ -238,7 +240,7 @@ class SpiderFoot:
ncounter = ncounter + 1
if dst in root:
col = "#f00"
- ret['nodes'].append({'id': str(ncounter),
+ ret['nodes'].append({'id': str(ncounter),
'label': unicode(dst, errors="replace"),
'x': random.randint(1,1000),
'y': random.randint(1,1000),
@@ -251,7 +253,7 @@ class SpiderFoot:
if src in root:
col = "#f00"
ncounter = ncounter + 1
- ret['nodes'].append({'id': str(ncounter),
+ ret['nodes'].append({'id': str(ncounter),
'label': unicode(src, errors="replace"),
'x': random.randint(1,1000),
'y': random.randint(1,1000),
@@ -261,8 +263,8 @@ class SpiderFoot:
nodelist[src] = ncounter
ecounter = ecounter + 1
- ret['edges'].append({'id': str(ecounter),
- 'source': str(nodelist[src]),
+ ret['edges'].append({'id': str(ecounter),
+ 'source': str(nodelist[src]),
'target': str(nodelist[dst])
})
@@ -606,7 +608,7 @@ class SpiderFoot:
for mod in self.opts['__modules__'].keys():
if self.opts['__modules__'][mod]['consumes'] is None:
continue
-
+
if "*" in self.opts['__modules__'][mod]['consumes'] and mod not in modlist:
modlist.append(mod)
@@ -772,7 +774,7 @@ class SpiderFoot:
def sanitiseInput(self, cmd):
chars = ['a','b','c','d','e','f','g','h','i','j','k','l','m',
'n','o','p','q','r','s','t','u','v','w','x','y','z',
- '0','1','2','3','4','5','6','7','8','9','-','.']
+ '0','1','2','3','4','5','6','7','8','9','-','.']
for c in cmd:
if c.lower() not in chars:
return False
@@ -1061,7 +1063,7 @@ class SpiderFoot:
# Fetch a URL, return the response object
def fetchUrl(self, url, fatal=False, cookies=None, timeout=30,
- useragent="SpiderFoot", headers=None, noLog=False,
+ useragent="SpiderFoot", headers=None, noLog=False,
postData=None, dontMangle=False, sizeLimit=None,
headOnly=False, verify=False):
result = {
@@ -1242,7 +1244,7 @@ class SpiderFoot:
if attempts == 3:
return None
-
+
returnResults[seedUrl] = firstPage['content']
pat = re.compile("(\/search\S+start=\d+.[^\'\"]*sa=N)", re.IGNORECASE)
matches = re.findall(pat, firstPage['content'])
@@ -1299,106 +1301,42 @@ class SpiderFoot:
return returnResults
- # Scrape Bing for content, starting at startUrl and iterating through
- # results based on options supplied. Will return a dictionary of all pages
- # fetched and their contents {page => content}.
+ # Request search results from bing api. Will return a dict:
+ # {
+ # "urls": a list of urls that match the query string,
+ # "webSearchUrl": url for bing results page,
+ # }
# Options accepted:
- # limit: number of search result pages before returning, default is 10
- # nopause: don't randomly pause between fetches
+ # count: number of search results to request from the api
# useragent: User-Agent string to use
- # timeout: Fetch timeout
+ # timeout: api call timeout
def bingIterate(self, searchString, opts=dict()):
- limit = 10
- fetches = 0
- returnResults = dict()
-
- if 'limit' in opts:
- limit = opts['limit']
-
- # We attempt to make the URL look as authentically human as possible
- seedUrl = u"http://www.bing.com/search?q={0}".format(searchString) + \
- u"&pc=MOZI"
-
- attempts = 0
- failed = False
- while attempts < 3:
- firstPage = self.fetchUrl(seedUrl, timeout=opts['timeout'],
- useragent=opts['useragent'])
- if firstPage['code'] == "400":
- self.error("Bing doesn't like us right now..", False)
- failed = True
-
- if firstPage['content'] is None:
- self.error("Failed to fetch content from Bing.", False)
- failed = True
- else:
- if "/challengepic?" in firstPage['content']:
- self.error("Bing returned a CAPTCHA.", False)
- failed = True
-
- if failed:
- self.refreshTorIdent()
- attempts += 1
- failed = False
- else:
- break
-
- if attempts == 3:
+ endpoint = "https://api.cognitive.microsoft.com/bing/v7.0/search"
+ # ToDO: figure out how to add credentials in config:
+ key = ""
+ response = requests.get(
+ endpoint,
+ timeout=opts["timeout"],
+ headers={"Ocp-Apim-Subscription-Key": key, "User-Agent": opts["useragent"]},
+ params={
+ "q": searchString,
+ "responseFilter": "Webpages",
+ "count": opts["count"],
+ },
+ )
+ if response.status_code != 200:
+ message = "Failed to talk to Bing API, status code: %s, response: %s".format(
+ response.status_code, response.text
+ )
+ self.error(message)
return None
- returnResults[seedUrl] = firstPage['content']
- pat = re.compile("(\/search\S+first=\d+.[^\'\"]*FORM=\S+)", re.IGNORECASE)
- matches = re.findall(pat, firstPage['content'])
- while matches > 0 and fetches < limit:
- nextUrl = None
- fetches += 1
- for match in matches:
- # Bing moves in increments of 10
- if "first=" + str((fetches * 10) + 1) in match:
- nextUrl = match.replace("&amp;", "&").replace("%3a", ":")
-
- if nextUrl is None:
- self.debug("Nothing left to scan for in Bing results.")
- return returnResults
- self.info("Next Bing URL: " + nextUrl)
-
- # Wait for a random number of seconds between fetches
- if 'nopause' not in opts:
- pauseSecs = random.randint(4, 15)
- self.info("Pausing for " + str(pauseSecs))
- time.sleep(pauseSecs)
-
- attempts = 0
- while attempts < 3:
- nextPage = self.fetchUrl(u'https://www.bing.com' + nextUrl,
- timeout=opts['timeout'], useragent=opts['useragent'])
- if nextPage['code'] == "400":
- self.error("Bing doesn't like us any more..", False)
- failed = True
-
- if nextPage['content'] is None:
- self.error("Failed to fetch subsequent content from Bing.", False)
- failed = True
- else:
- if "/challengepic?" in firstPage['content']:
- self.error("Bing returned a CAPTCHA.", False)
- failed = True
-
- if failed:
- self.refreshTorIdent()
- attempts += 1
- failed = False
- else:
- break
-
- if attempts == 3:
- return returnResults
-
- returnResults[nextUrl] = nextPage['content']
- pat = re.compile("(\/search\S+first=\d+.[^\'\"]*)", re.IGNORECASE)
- matches = re.findall(pat, nextPage['content'])
+ results = {
+ "urls": [result["url"] for result in response.json()["webPages"]["value"]],
+ "webSearchUrl": response.json()["webPages"]["webSearchUrl"],
+ }
+ return results
- return returnResults
# SpiderFoot plug-in module base class
#
@@ -1559,7 +1497,7 @@ class SpiderFootPlugin(object):
except BaseException as e:
f = open("sferror.log", "a")
f.write("Module (" + listener.__module__ + ") encountered an error: " + str(e) + "\n")
-
+
exc_type, exc_value, exc_traceback = sys.exc_info()
f.write(repr(traceback.format_exception(exc_type, exc_value, exc_traceback)))
f.close()
@@ -1711,7 +1649,7 @@ class SpiderFootTarget(object):
# 2.1
if value == name:
return True
- # 2.2
+ # 2.2
if includeParents and name.endswith("." + value):
return True
# 2.3