SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.
Python attacksurfacecticybersecurityfootprintinghackinginformation-gatheringinformation-securityinfosecintelligence-gatheringosintosint-frameworkosint-reconnaissanceosint-toolpentestingpythonreconsecurity-toolsthreat-intelligencethreatintel
Re-factoring due to split up of modules.
Steve0975a804414d
diff --git a/modules/sfp_pastebin.py b/modules/sfp_pastebin.pyindex 2f4a49e6..8b6a1464 100644--- a/modules/sfp_pastebin.py+++ b/modules/sfp_pastebin.py@@ -62,65 +62,64 @@ class sfp_pastebin(SpiderFootPlugin):self.results.append(eventData)for dom in self.domains.keys():- if self.opts[dom]:- links = list()- target = self.domains[dom]- pages = self.sf.googleIterate("site:" + target + "%20+\"" + eventData + "\"",- dict(limit=self.opts['searchpages'],- useragent=self.opts['_useragent'],- timeout=self.opts['_fetchtimeout']))-- if pages is None:- self.sf.info("No results returned from Google search of " + target + ".")+ links = list()+ target = self.domains[dom]+ pages = self.sf.googleIterate("site:" + target + "%20+\"" + eventData + "\"",+ dict(limit=self.opts['searchpages'],+ useragent=self.opts['_useragent'],+ timeout=self.opts['_fetchtimeout']))++ if pages is None:+ self.sf.info("No results returned from Google search of " + target + ".")+ return None++ for page in pages.keys():+ if page in self.results:+ continue+ else:+ self.results.append(page)++ # Check if we've been asked to stop+ if self.checkForStop():return None- for page in pages.keys():- if page in self.results:- continue- else:- self.results.append(page)+ # Fetch the paste site content+ links += self.sf.parseLinks(page, pages[page], target)++ for link in links:+ if link in self.results:+ continue+ else:+ self.results.append(link)- # Check if we've been asked to stop+ self.sf.debug("Found a link: " + link)+ if self.sf.urlBaseUrl(link).endswith(target):if self.checkForStop():return None- # Fetch the paste site content- links += self.sf.parseLinks(page, pages[page], target)+ res = self.sf.fetchUrl(link, timeout=self.opts['_fetchtimeout'],+ useragent=self.opts['_useragent'])- for link in links:- if link in self.results:+ if res['content'] is None:+ self.sf.debug("Ignoring " + link + " as no data returned")continue- else:- self.results.append(link)-- self.sf.debug("Found a link: " + link)- if self.sf.urlBaseUrl(link).endswith(target):- if self.checkForStop():- return None- res = self.sf.fetchUrl(link, timeout=self.opts['_fetchtimeout'],- useragent=self.opts['_useragent'])+ # Sometimes pastes search results false positives+ if re.search("[^a-zA-Z\-\_0-9]" + re.escape(eventData) ++ "[^a-zA-Z\-\_0-9]", res['content'], re.IGNORECASE) is None:+ continue- if res['content'] is None:- self.sf.debug("Ignoring " + link + " as no data returned")- continue-- # Sometimes pastes search results false positives- if re.search("[^a-zA-Z\-\_0-9]" + re.escape(eventData) +- "[^a-zA-Z\-\_0-9]", res['content'], re.IGNORECASE) is None:- continue-- try:- startIndex = res['content'].index(eventData)- except BaseException as e:- self.sf.debug("String not found in pastes content.")- continue+ try:+ startIndex = res['content'].index(eventData)+ except BaseException as e:+ self.sf.debug("String not found in pastes content.")+ continue- evt1 = SpiderFootEvent("LEAKSITE_URL", link, self.__name__, event)- self.notifyListeners(evt1)+ evt1 = SpiderFootEvent("LEAKSITE_URL", link, self.__name__, event)+ self.notifyListeners(evt1)- evt2 = SpiderFootEvent("LEAKSITE_CONTENT", res['content'], self.__name__, evt1)- self.notifyListeners(evt2)+ evt2 = SpiderFootEvent("LEAKSITE_CONTENT", res['content'], self.__name__, evt1)+ self.notifyListeners(evt2)# End of sfp_pastebin classdiff --git a/modules/sfp_pastie.py b/modules/sfp_pastie.pyindex 0dac6783..53ae067d 100644--- a/modules/sfp_pastie.py+++ b/modules/sfp_pastie.py@@ -62,65 +62,64 @@ class sfp_pastie(SpiderFootPlugin):self.results.append(eventData)for dom in self.domains.keys():- if self.opts[dom]:- links = list()- target = self.domains[dom]- pages = self.sf.googleIterate("site:" + target + "%20+\"" + eventData + "\"",- dict(limit=self.opts['searchpages'],- useragent=self.opts['_useragent'],- timeout=self.opts['_fetchtimeout']))-- if pages is None:- self.sf.info("No results returned from Google search of " + target + ".")+ links = list()+ target = self.domains[dom]+ pages = self.sf.googleIterate("site:" + target + "%20+\"" + eventData + "\"",+ dict(limit=self.opts['searchpages'],+ useragent=self.opts['_useragent'],+ timeout=self.opts['_fetchtimeout']))++ if pages is None:+ self.sf.info("No results returned from Google search of " + target + ".")+ return None++ for page in pages.keys():+ if page in self.results:+ continue+ else:+ self.results.append(page)++ # Check if we've been asked to stop+ if self.checkForStop():return None- for page in pages.keys():- if page in self.results:- continue- else:- self.results.append(page)+ # Fetch the paste site content+ links += self.sf.parseLinks(page, pages[page], target)++ for link in links:+ if link in self.results:+ continue+ else:+ self.results.append(link)- # Check if we've been asked to stop+ self.sf.debug("Found a link: " + link)+ if self.sf.urlBaseUrl(link).endswith(target):if self.checkForStop():return None- # Fetch the paste site content- links += self.sf.parseLinks(page, pages[page], target)+ res = self.sf.fetchUrl(link, timeout=self.opts['_fetchtimeout'],+ useragent=self.opts['_useragent'])- for link in links:- if link in self.results:+ if res['content'] is None:+ self.sf.debug("Ignoring " + link + " as no data returned")continue- else:- self.results.append(link)-- self.sf.debug("Found a link: " + link)- if self.sf.urlBaseUrl(link).endswith(target):- if self.checkForStop():- return None- res = self.sf.fetchUrl(link, timeout=self.opts['_fetchtimeout'],- useragent=self.opts['_useragent'])+ # Sometimes pastes search results false positives+ if re.search("[^a-zA-Z\-\_0-9]" + re.escape(eventData) ++ "[^a-zA-Z\-\_0-9]", res['content'], re.IGNORECASE) is None:+ continue- if res['content'] is None:- self.sf.debug("Ignoring " + link + " as no data returned")- continue-- # Sometimes pastes search results false positives- if re.search("[^a-zA-Z\-\_0-9]" + re.escape(eventData) +- "[^a-zA-Z\-\_0-9]", res['content'], re.IGNORECASE) is None:- continue-- try:- startIndex = res['content'].index(eventData)- except BaseException as e:- self.sf.debug("String not found in pastes content.")- continue+ try:+ startIndex = res['content'].index(eventData)+ except BaseException as e:+ self.sf.debug("String not found in pastes content.")+ continue- evt1 = SpiderFootEvent("LEAKSITE_URL", link, self.__name__, event)- self.notifyListeners(evt1)+ evt1 = SpiderFootEvent("LEAKSITE_URL", link, self.__name__, event)+ self.notifyListeners(evt1)- evt2 = SpiderFootEvent("LEAKSITE_CONTENT", res['content'], self.__name__, evt1)- self.notifyListeners(evt2)+ evt2 = SpiderFootEvent("LEAKSITE_CONTENT", res['content'], self.__name__, evt1)+ self.notifyListeners(evt2)# End of sfp_pastie class