SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.
Python attacksurfacecticybersecurityfootprintinghackinginformation-gatheringinformation-securityinfosecintelligence-gatheringosintosint-frameworkosint-reconnaissanceosint-toolpentestingpythonreconsecurity-toolsthreat-intelligencethreatintel
feat: update 'googleIterate' & googlesearch module
Amr Awad42146a110b8f
diff --git a/modules/sfp_bingsearch.py b/modules/sfp_bingsearch.pyindex b19be908..07889d35 100644--- a/modules/sfp_bingsearch.py+++ b/modules/sfp_bingsearch.py@@ -67,6 +67,7 @@ class sfp_bingsearch(SpiderFootPlugin):if results is None:# Failed to talk to the bing API or no results returnedreturn None+urls = results["urls"]new_links = list(set(urls) - set(self.results))diff --git a/modules/sfp_googlesearch.py b/modules/sfp_googlesearch.pyindex adbbe7e4..7d33b1ea 100644--- a/modules/sfp_googlesearch.py+++ b/modules/sfp_googlesearch.py@@ -10,21 +10,24 @@# Licence: GPL# -------------------------------------------------------------------------------+import urllib+from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEventclass sfp_googlesearch(SpiderFootPlugin):- """Google:Footprint,Investigate,Passive:Search Engines:errorprone:Some light Google scraping to identify sub-domains and links."""+ """Google:Footprint,Investigate,Passive:Search Engines:apikey:Obtain information from the Google Custom Search API to identify sub-domains and links."""# Default optionsopts = {- 'pages': 20 # Number of google results pages to iterate+ "api_key": "", "cse_id": "",}# Option descriptionsoptdescs = {- 'pages': "Number of Google results pages to iterate through."+ "api_key": "Google API Key.",+ "cse_id": "Google Custom Search Engine ID.",}# Target@@ -59,44 +62,48 @@ class sfp_googlesearch(SpiderFootPlugin):self.results.append(eventData)# Sites hosted on the domain- pages = self.sf.googleIterate("site:" + eventData,- dict(limit=self.opts['pages'], useragent=self.opts['_useragent'],- timeout=self.opts['_fetchtimeout']))- if pages is None:- self.sf.info("No results returned from Google.")+ results = self.sf.googleIterate(+ searchString="site:" + eventData,+ opts={+ "timeout": self.opts["_fetchtimeout"],+ "useragent": self.opts["_useragent"],+ "api_key": self.opts["api_key"],+ "cse_id": self.opts["cse_id"],+ },+ )+ if results is None:+ # Failed to talk to the bing API or no results returnedreturn None- for page in pages.keys():- found = False- if page in self.results:- continue- else:- self.results.append(page)-- links = self.sf.parseLinks(page, pages[page], eventData)- if len(links) == 0:- continue-- for link in links:- if self.checkForStop():- return None-- if link in self.results:- continue- else:- self.results.append(link)- self.sf.debug("Found a link: " + link)- if self.sf.urlFQDN(link).endswith(eventData):- found = True- evt = SpiderFootEvent("LINKED_URL_INTERNAL", link,- self.__name__, event)- self.notifyListeners(evt)-- if found:- # Submit the google results for analysis- evt = SpiderFootEvent("SEARCH_ENGINE_WEB_CONTENT", pages[page],- self.__name__, event)+ urls = results["urls"]+ new_links = list(set(urls) - set(self.results))++ # Add new links to results+ self.results.extend(new_links)++ internal_links = [+ link for link in new_links if self.sf.urlFQDN(link).endswith(eventData)+ ]+ for link in internal_links:+ self.sf.debug("Found a link: " + link)++ evt = SpiderFootEvent("LINKED_URL_INTERNAL", link, self.__name__, event)+ self.notifyListeners(evt)++ if internal_links:+ # Submit the bing results for analysis+ bingsearch_url = results["webSearchUrl"]+ response = self.sf.fetchUrl(+ bingsearch_url,+ timeout=self.opts["_fetchtimeout"],+ useragent=self.opts["_useragent"],+ )+ if response['status'] == 'OK':+ evt = SpiderFootEvent(+ "SEARCH_ENGINE_WEB_CONTENT", response["content"], self.__name__, event+ )self.notifyListeners(evt)-+ else:+ self.sf.error("Failed to fetch bing web search URL", exception=False)# End of sfp_googlesearch classdiff --git a/sflib.py b/sflib.pyindex bcf01f08..74fd0e81 100644--- a/sflib.py+++ b/sflib.py@@ -1199,110 +1199,54 @@ class SpiderFoot:except BaseException as e:return False- # Scrape Google for content, starting at startUrl and iterating through- # results based on options supplied. Will return a dictionary of all pages- # fetched and their contents {page => content}.+ # Request search results from the Google API. Will return a dict:+ # {+ # "urls": a list of urls that match the query string,+ # "webSearchUrl": url for Google results page,+ # }# Options accepted:- # limit: number of search result pages before returning, default is 10- # nopause: don't randomly pause between fetches# useragent: User-Agent string to use- # timeout: Fetch timeout+ # timeout: API call timeoutdef googleIterate(self, searchString, opts=dict()):- limit = 10- fetches = 0- returnResults = dict()-- if 'limit' in opts:- limit = opts['limit']-- # We attempt to make the URL look as authentically human as possible- seedUrl = u"https://www.google.com/search?q={0}".format(searchString) + \- u"&ie=utf-8&oe=utf-8&aq=t&rls=org.mozilla:en-US:official&client=firefox-a"-- attempts = 0- failed = False- while attempts < 3:- firstPage = self.fetchUrl(seedUrl, timeout=opts['timeout'],- useragent=opts['useragent'])- if firstPage['code'] == "403" or firstPage['code'] == "503":- self.error("Google doesn't like us right now..", False)- failed = True-- if firstPage['content'] is None:- self.error("Failed to fetch content from Google.", False)- failed = True- else:- if "name=\"captcha\"" in firstPage['content']:- self.error("Google returned a CAPTCHA.", False)- failed = True-- if failed:- self.refreshTorIdent()- attempts += 1- failed = False- else:- break+ endpoint = "https://www.googleapis.com/customsearch/v1?"+ params = {+ "q": searchString,+ "cx": opts["cse_id"],+ "key": opts["api_key"],+ }++ response = self.fetchUrl(+ endpoint + urllib.urlencode(params),+ timeout=opts["timeout"],+ useragent=opts["useragent"],+ )- if attempts == 3:+ if response['status'] != 'OK':+ self.error("Failed to get a valid response from the Google API", exception=False)return None- returnResults[seedUrl] = firstPage['content']- pat = re.compile("(\/search\S+start=\d+.[^\'\"]*sa=N)", re.IGNORECASE)- matches = re.findall(pat, firstPage['content'])-- while matches > 0 and fetches < limit:- nextUrl = None- fetches += 1- for match in matches:- # Google moves in increments of 10- if "start=" + str(fetches * 10) in match:- nextUrl = match.replace("&", "&")-- if nextUrl is None:- self.debug("Nothing left to scan for in Google results.")- return returnResults- self.info("Next Google URL: " + nextUrl)-- # Wait for a random number of seconds between fetches- if 'nopause' not in opts:- pauseSecs = random.randint(4, 15)- self.info("Pausing for " + str(pauseSecs))- time.sleep(pauseSecs)-- attempts = 0- failed = False- while attempts < 3:- nextPage = self.fetchUrl(u'https://www.google.com' + nextUrl,- timeout=opts['timeout'], useragent=opts['useragent'])- if nextPage['code'] == "403" or nextPage['code'] == "503":- self.error("Google doesn't like us right now..", False)- failed = True-- if nextPage['content'] is None:- self.error("Failed to fetch subsequent content from Google.", False)- failed = True- else:- if "name=\"captcha\"" in nextPage['content']:- self.error("Google returned a CAPTCHA.", False)- failed = True-- if failed:- self.refreshTorIdent()- attempts += 1- failed = False- else:- break+ try:+ response_json = json.loads(response['content'])+ except ValueError:+ self.error("the key 'content' in the Google API response doesn't contain valid json.", exception=False)+ return None- if attempts == 3:- return returnResults+ if "items" in response_json:+ # We attempt to make the URL look as authentically human as possible+ search_url = u"https://www.google.com/search?q={0}".format(searchString) + \+ u"&ie=utf-8&oe=utf-8&aq=t&rls=org.mozilla:en-US:official&client=firefox-a"- returnResults[nextUrl] = nextPage['content']- pat = re.compile("(\/search\S+start=\d+.[^\'\"]*)", re.IGNORECASE)- matches = re.findall(pat, nextPage['content'])+ results = {+ "urls": [str(k['link']) for k in response_json['items']],+ "webSearchUrl": search_url,+ }+ else:+ return None++ return results- return returnResults- # Request search results from the bing API. Will return a dict:+ # Request search results from the Bing API. Will return a dict:# {# "urls": a list of urls that match the query string,# "webSearchUrl": url for bing results page,