Skip to content

SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.

feat: update 'googleIterate' & googlesearch module

Amr Awad42146a110b8f
diff --git a/modules/sfp_bingsearch.py b/modules/sfp_bingsearch.py
index b19be908..07889d35 100644
--- a/modules/sfp_bingsearch.py
+++ b/modules/sfp_bingsearch.py
@@ -67,6 +67,7 @@ class sfp_bingsearch(SpiderFootPlugin):
if results is None:
# Failed to talk to the bing API or no results returned
return None
+
urls = results["urls"]
new_links = list(set(urls) - set(self.results))
diff --git a/modules/sfp_googlesearch.py b/modules/sfp_googlesearch.py
index adbbe7e4..7d33b1ea 100644
--- a/modules/sfp_googlesearch.py
+++ b/modules/sfp_googlesearch.py
@@ -10,21 +10,24 @@
# Licence: GPL
# -------------------------------------------------------------------------------
+import urllib
+
from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
class sfp_googlesearch(SpiderFootPlugin):
- """Google:Footprint,Investigate,Passive:Search Engines:errorprone:Some light Google scraping to identify sub-domains and links."""
+ """Google:Footprint,Investigate,Passive:Search Engines:apikey:Obtain information from the Google Custom Search API to identify sub-domains and links."""
# Default options
opts = {
- 'pages': 20 # Number of google results pages to iterate
+ "api_key": "", "cse_id": "",
}
# Option descriptions
optdescs = {
- 'pages': "Number of Google results pages to iterate through."
+ "api_key": "Google API Key.",
+ "cse_id": "Google Custom Search Engine ID.",
}
# Target
@@ -59,44 +62,48 @@ class sfp_googlesearch(SpiderFootPlugin):
self.results.append(eventData)
# Sites hosted on the domain
- pages = self.sf.googleIterate("site:" + eventData,
- dict(limit=self.opts['pages'], useragent=self.opts['_useragent'],
- timeout=self.opts['_fetchtimeout']))
- if pages is None:
- self.sf.info("No results returned from Google.")
+ results = self.sf.googleIterate(
+ searchString="site:" + eventData,
+ opts={
+ "timeout": self.opts["_fetchtimeout"],
+ "useragent": self.opts["_useragent"],
+ "api_key": self.opts["api_key"],
+ "cse_id": self.opts["cse_id"],
+ },
+ )
+ if results is None:
+ # Failed to talk to the bing API or no results returned
return None
- for page in pages.keys():
- found = False
- if page in self.results:
- continue
- else:
- self.results.append(page)
-
- links = self.sf.parseLinks(page, pages[page], eventData)
- if len(links) == 0:
- continue
-
- for link in links:
- if self.checkForStop():
- return None
-
- if link in self.results:
- continue
- else:
- self.results.append(link)
- self.sf.debug("Found a link: " + link)
- if self.sf.urlFQDN(link).endswith(eventData):
- found = True
- evt = SpiderFootEvent("LINKED_URL_INTERNAL", link,
- self.__name__, event)
- self.notifyListeners(evt)
-
- if found:
- # Submit the google results for analysis
- evt = SpiderFootEvent("SEARCH_ENGINE_WEB_CONTENT", pages[page],
- self.__name__, event)
+ urls = results["urls"]
+ new_links = list(set(urls) - set(self.results))
+
+ # Add new links to results
+ self.results.extend(new_links)
+
+ internal_links = [
+ link for link in new_links if self.sf.urlFQDN(link).endswith(eventData)
+ ]
+ for link in internal_links:
+ self.sf.debug("Found a link: " + link)
+
+ evt = SpiderFootEvent("LINKED_URL_INTERNAL", link, self.__name__, event)
+ self.notifyListeners(evt)
+
+ if internal_links:
+ # Submit the bing results for analysis
+ bingsearch_url = results["webSearchUrl"]
+ response = self.sf.fetchUrl(
+ bingsearch_url,
+ timeout=self.opts["_fetchtimeout"],
+ useragent=self.opts["_useragent"],
+ )
+ if response['status'] == 'OK':
+ evt = SpiderFootEvent(
+ "SEARCH_ENGINE_WEB_CONTENT", response["content"], self.__name__, event
+ )
self.notifyListeners(evt)
-
+ else:
+ self.sf.error("Failed to fetch bing web search URL", exception=False)
# End of sfp_googlesearch class
diff --git a/sflib.py b/sflib.py
index bcf01f08..74fd0e81 100644
--- a/sflib.py
+++ b/sflib.py
@@ -1199,110 +1199,54 @@ class SpiderFoot:
except BaseException as e:
return False
- # Scrape Google for content, starting at startUrl and iterating through
- # results based on options supplied. Will return a dictionary of all pages
- # fetched and their contents {page => content}.
+ # Request search results from the Google API. Will return a dict:
+ # {
+ # "urls": a list of urls that match the query string,
+ # "webSearchUrl": url for Google results page,
+ # }
# Options accepted:
- # limit: number of search result pages before returning, default is 10
- # nopause: don't randomly pause between fetches
# useragent: User-Agent string to use
- # timeout: Fetch timeout
+ # timeout: API call timeout
def googleIterate(self, searchString, opts=dict()):
- limit = 10
- fetches = 0
- returnResults = dict()
-
- if 'limit' in opts:
- limit = opts['limit']
-
- # We attempt to make the URL look as authentically human as possible
- seedUrl = u"https://www.google.com/search?q={0}".format(searchString) + \
- u"&ie=utf-8&oe=utf-8&aq=t&rls=org.mozilla:en-US:official&client=firefox-a"
-
- attempts = 0
- failed = False
- while attempts < 3:
- firstPage = self.fetchUrl(seedUrl, timeout=opts['timeout'],
- useragent=opts['useragent'])
- if firstPage['code'] == "403" or firstPage['code'] == "503":
- self.error("Google doesn't like us right now..", False)
- failed = True
-
- if firstPage['content'] is None:
- self.error("Failed to fetch content from Google.", False)
- failed = True
- else:
- if "name=\"captcha\"" in firstPage['content']:
- self.error("Google returned a CAPTCHA.", False)
- failed = True
-
- if failed:
- self.refreshTorIdent()
- attempts += 1
- failed = False
- else:
- break
+ endpoint = "https://www.googleapis.com/customsearch/v1?"
+ params = {
+ "q": searchString,
+ "cx": opts["cse_id"],
+ "key": opts["api_key"],
+ }
+
+ response = self.fetchUrl(
+ endpoint + urllib.urlencode(params),
+ timeout=opts["timeout"],
+ useragent=opts["useragent"],
+ )
- if attempts == 3:
+ if response['status'] != 'OK':
+ self.error("Failed to get a valid response from the Google API", exception=False)
return None
- returnResults[seedUrl] = firstPage['content']
- pat = re.compile("(\/search\S+start=\d+.[^\'\"]*sa=N)", re.IGNORECASE)
- matches = re.findall(pat, firstPage['content'])
-
- while matches > 0 and fetches < limit:
- nextUrl = None
- fetches += 1
- for match in matches:
- # Google moves in increments of 10
- if "start=" + str(fetches * 10) in match:
- nextUrl = match.replace("&amp;", "&")
-
- if nextUrl is None:
- self.debug("Nothing left to scan for in Google results.")
- return returnResults
- self.info("Next Google URL: " + nextUrl)
-
- # Wait for a random number of seconds between fetches
- if 'nopause' not in opts:
- pauseSecs = random.randint(4, 15)
- self.info("Pausing for " + str(pauseSecs))
- time.sleep(pauseSecs)
-
- attempts = 0
- failed = False
- while attempts < 3:
- nextPage = self.fetchUrl(u'https://www.google.com' + nextUrl,
- timeout=opts['timeout'], useragent=opts['useragent'])
- if nextPage['code'] == "403" or nextPage['code'] == "503":
- self.error("Google doesn't like us right now..", False)
- failed = True
-
- if nextPage['content'] is None:
- self.error("Failed to fetch subsequent content from Google.", False)
- failed = True
- else:
- if "name=\"captcha\"" in nextPage['content']:
- self.error("Google returned a CAPTCHA.", False)
- failed = True
-
- if failed:
- self.refreshTorIdent()
- attempts += 1
- failed = False
- else:
- break
+ try:
+ response_json = json.loads(response['content'])
+ except ValueError:
+ self.error("the key 'content' in the Google API response doesn't contain valid json.", exception=False)
+ return None
- if attempts == 3:
- return returnResults
+ if "items" in response_json:
+ # We attempt to make the URL look as authentically human as possible
+ search_url = u"https://www.google.com/search?q={0}".format(searchString) + \
+ u"&ie=utf-8&oe=utf-8&aq=t&rls=org.mozilla:en-US:official&client=firefox-a"
- returnResults[nextUrl] = nextPage['content']
- pat = re.compile("(\/search\S+start=\d+.[^\'\"]*)", re.IGNORECASE)
- matches = re.findall(pat, nextPage['content'])
+ results = {
+ "urls": [str(k['link']) for k in response_json['items']],
+ "webSearchUrl": search_url,
+ }
+ else:
+ return None
+
+ return results
- return returnResults
- # Request search results from the bing API. Will return a dict:
+ # Request search results from the Bing API. Will return a dict:
# {
# "urls": a list of urls that match the query string,
# "webSearchUrl": url for bing results page,