SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.
Python attacksurfacecticybersecurityfootprintinghackinginformation-gatheringinformation-securityinfosecintelligence-gatheringosintosint-frameworkosint-reconnaissanceosint-toolpentestingpythonreconsecurity-toolsthreat-intelligencethreatintel
Added pastie.org and notepad.cc based on changes from @Wingless-Archangel.
smicallefdad4a9f62532
diff --git a/modules/sfp_pastebin.py b/modules/sfp_pastebin.pydeleted file mode 100644index 54374e57..00000000--- a/modules/sfp_pastebin.py+++ /dev/null@@ -1,130 +0,0 @@-# -*- coding: utf-8 -*--# --------------------------------------------------------------------------------# Name: sfp_pastebin-# Purpose: Searches Google for PasteBin content related to the domain in-# question.-#-# Author: Steve Micallef <[email protected]>-#-# Created: 20/03/2014-# Copyright: (c) Steve Micallef 2014-# Licence: GPL-# ---------------------------------------------------------------------------------import re-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent---class sfp_pastebin(SpiderFootPlugin):- """PasteBin:PasteBin scraping (via Google) to identify related content."""-- # Default options- opts = {- 'pages': 20 # Number of google results pages to iterate- }-- # Option descriptions- optdescs = {- 'pages': "Number of search results pages to iterate through."- }-- results = list()-- def setup(self, sfc, userOpts=dict()):- self.sf = sfc- self.results = list()-- for opt in userOpts.keys():- self.opts[opt] = userOpts[opt]-- # What events is this module interested in for input- def watchedEvents(self):- return ["INTERNET_NAME", "IP_ADDRESS", "EMAILADDR"]-- # What events this module produces- # This is to support the end user in selecting modules based on events- # produced.- def producedEvents(self):- return ["SEARCH_ENGINE_WEB_CONTENT", "PASTEBIN_CONTENT"]-- def handleEvent(self, event):- eventName = event.eventType- srcModuleName = event.module- eventData = event.data-- if eventData in self.results:- return None- else:- self.results.append(eventData)-- # Sites hosted on the domain- pages = self.sf.googleIterate("site:pastebin.com +\"" + eventData + "\"",- dict(limit=self.opts['pages'],- useragent=self.opts['_useragent'], timeout=self.opts['_fetchtimeout']))-- if pages is None:- self.sf.info("No results returned from Google PasteBin search.")- return None-- for page in pages.keys():- if page in self.results:- continue- else:- self.results.append(page)-- # Check if we've been asked to stop- if self.checkForStop():- return None-- # Submit the google results for analysis- evt = SpiderFootEvent("SEARCH_ENGINE_WEB_CONTENT", pages[page],- self.__name__, event)- self.notifyListeners(evt)-- # Fetch the PasteBin page- links = self.sf.parseLinks(page, pages[page], "pastebin.com")- if len(links) == 0:- continue-- for link in links:- if link in self.results:- continue- else:- self.results.append(link)-- self.sf.debug("Found a link: " + link)- if self.sf.urlBaseUrl(link).endswith("pastebin.com"):- if self.checkForStop():- return None-- res = self.sf.fetchUrl(link, timeout=self.opts['_fetchtimeout'],- useragent=self.opts['_useragent'])-- if res['content'] is None:- self.sf.debug("Ignoring " + link + " as no data returned")- continue-- evt = SpiderFootEvent("SEARCH_ENGINE_WEB_CONTENT",- res['content'], self.__name__, event)- self.notifyListeners(evt)-- # Sometimes pastebin search results false positives- if re.search("[^a-zA-Z\-\_0-9]" + re.escape(eventData) +- "[^a-zA-Z\-\_0-9]", res['content'], re.IGNORECASE) is None:- continue-- try:- startIndex = res['content'].index(eventData) - 120- endIndex = startIndex + len(eventData) + 240- except BaseException as e:- self.sf.debug("String not found in pastebin content.")- continue-- data = res['content'][startIndex:endIndex]-- evt = SpiderFootEvent("PASTEBIN_CONTENT",- "<SFURL>" + link + "</SFURL>\n" + "\"... " + data + " ...\"",- self.__name__, event)- self.notifyListeners(evt)--# End of sfp_pastebin classdiff --git a/modules/sfp_pastes.py b/modules/sfp_pastes.pynew file mode 100644index 00000000..a04927b2--- /dev/null+++ b/modules/sfp_pastes.py@@ -0,0 +1,144 @@+# -*- coding: utf-8 -*-+# -------------------------------------------------------------------------------+# Name: sfp_pastes+# Purpose: Searches Google for PasteBin content related to the domain in+# question.+#+# Author: Steve Micallef <[email protected]> and ShellCodeNoobx+#+# Created: 20/03/2014+# Copyright: (c) Steve Micallef 2014+# Licence: GPL+# -------------------------------------------------------------------------------++import re+from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+++class sfp_pastes(SpiderFootPlugin):+ """Pastes:PasteBin, Pastie and Notepad.cc scraping (via Google) to identify related content."""++ # Default options+ opts = {+ 'searchpages': 20, # Number of google results pages to iterate+ 'pastebin': True,+ 'pastie': True,+ 'notepadcc': True+ }++ # Option descriptions+ optdescs = {+ 'searchpages': "Number of search results pages to iterate through.",+ 'pastebin': "Search PasteBin?",+ 'pastie': "Search Pastie?",+ 'notepadcc': "Search Notepad.cc?"+ }++ domains = {+ 'pastebin': "pastebin.com",+ 'pastie': "pastie.org",+ 'notepadcc': "notepadd.cc"+ }++ results = list()++ def setup(self, sfc, userOpts=dict()):+ self.sf = sfc+ self.results = list()++ for opt in userOpts.keys():+ self.opts[opt] = userOpts[opt]++ # What events is this module interested in for input+ def watchedEvents(self):+ return ["DOMAIN_NAME", "EMAILADDR"]++ # What events this module produces+ # This is to support the end user in selecting modules based on events+ # produced.+ def producedEvents(self):+ return ["SEARCH_ENGINE_WEB_CONTENT", "PASTESITE_CONTENT"]++ def handleEvent(self, event):+ eventName = event.eventType+ srcModuleName = event.module+ eventData = event.data++ if eventData in self.results:+ return None+ else:+ self.results.append(eventData)++ for dom in self.domains.keys():+ if self.opts[dom]:+ links = list()+ target = self.domains[dom]+ pages = self.sf.googleIterate("site:" + target + "%20+\"" + eventData + "\"",+ dict(limit=self.opts['searchpages'],+ useragent=self.opts['_useragent'],+ timeout=self.opts['_fetchtimeout']))++ if pages is None:+ self.sf.info("No results returned from Google search of " + target + ".")+ return None++ for page in pages.keys():+ if page in self.results:+ continue+ else:+ self.results.append(page)++ # Check if we've been asked to stop+ if self.checkForStop():+ return None++ # Submit the google results for analysis+ evt = SpiderFootEvent("SEARCH_ENGINE_WEB_CONTENT", pages[page],+ self.__name__, event)+ self.notifyListeners(evt)++ # Fetch the paste site content+ links += self.sf.parseLinks(page, pages[page], target)++ for link in links:+ if link in self.results:+ continue+ else:+ self.results.append(link)++ self.sf.debug("Found a link: " + link)+ if self.sf.urlBaseUrl(link).endswith(target):+ if self.checkForStop():+ return None++ res = self.sf.fetchUrl(link, timeout=self.opts['_fetchtimeout'],+ useragent=self.opts['_useragent'])++ if res['content'] is None:+ self.sf.debug("Ignoring " + link + " as no data returned")+ continue++ evt = SpiderFootEvent("SEARCH_ENGINE_WEB_CONTENT",+ res['content'], self.__name__, event)+ self.notifyListeners(evt)++ # Sometimes pastes search results false positives+ if re.search("[^a-zA-Z\-\_0-9]" + re.escape(eventData) ++ "[^a-zA-Z\-\_0-9]", res['content'], re.IGNORECASE) is None:+ continue++ try:+ startIndex = res['content'].index(eventData) - 120+ endIndex = startIndex + len(eventData) + 240+ except BaseException as e:+ self.sf.debug("String not found in pastes content.")+ continue++ data = res['content'][startIndex:endIndex]++ evt = SpiderFootEvent("PASTESITE_CONTENT",+ "<SFURL>" + link + "</SFURL>\n" + "\"... " + data + " ...\"",+ self.__name__, event)+ self.notifyListeners(evt)++# End of sfp_pastes class