Skip to content

SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.

feat: updated module pastebin

Amr Awadc4d0bb1b61fd
diff --git a/modules/sfp_pastebin.py b/modules/sfp_pastebin.py
index c7fb3db0..3f8f0689 100644
--- a/modules/sfp_pastebin.py
+++ b/modules/sfp_pastebin.py
@@ -16,17 +16,18 @@ from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
class sfp_pastebin(SpiderFootPlugin):
- """PasteBin:Footprint,Investigate,Passive:Leaks, Dumps and Breaches:errorprone:PasteBin scraping (via Google) to identify related content."""
+ """PasteBin:Footprint,Investigate,Passive:Leaks, Dumps and Breaches:apikey:PasteBin scraping (via Google) to identify related content."""
# Default options
opts = {
- 'searchpages': 20 # Number of google results pages to iterate
+ "api_key": "", "cse_id": "",
}
# Option descriptions
optdescs = {
- 'searchpages': "Number of search results pages to iterate through."
+ "api_key": "Google API Key.",
+ "cse_id": "Google Custom Search Engine ID.",
}
domains = {
@@ -65,62 +66,58 @@ class sfp_pastebin(SpiderFootPlugin):
for dom in self.domains.keys():
links = list()
target = self.domains[dom]
- pages = self.sf.googleIterate("site:" + target + "%20+\"" + eventData + "\"",
- dict(limit=self.opts['searchpages'],
- useragent=self.opts['_useragent'],
- timeout=self.opts['_fetchtimeout']))
-
- if pages is None:
- self.sf.info("No results returned from Google search of " + target + ".")
+ results = self.sf.googleIterate(
+ searchString="site:" + target + "%20+\"" + eventData + "\"",
+ opts={
+ "timeout": self.opts["_fetchtimeout"],
+ "useragent": self.opts["_useragent"],
+ "api_key": self.opts["api_key"],
+ "cse_id": self.opts["cse_id"],
+ },
+ )
+ if results is None:
+ # Failed to talk to the Google API or no results returned
return None
- for page in pages.keys():
- if page in self.results:
- continue
- else:
- self.results.append(page)
+ urls = results["urls"]
+ new_links = list(set(urls) - set(self.results))
+
+ # Add new links to results
+ self.results.extend(new_links)
+
+ relevant_links = [
+ link for link in new_links if self.sf.urlBaseUrl(link).endswith(target)
+ ]
+
+ for link in relevant_links:
+ self.sf.debug("Found a link: " + link)
- # Check if we've been asked to stop
if self.checkForStop():
return None
- # Fetch the paste site content
- links += self.sf.parseLinks(page, pages[page], target)
+ res = self.sf.fetchUrl(link, timeout=self.opts['_fetchtimeout'],
+ useragent=self.opts['_useragent'])
- for link in links:
- if link in self.results:
+ if res['content'] is None:
+ self.sf.debug("Ignoring " + link + " as no data returned")
continue
- else:
- self.results.append(link)
- self.sf.debug("Found a link: " + link)
- if self.sf.urlBaseUrl(link).endswith(target):
- if self.checkForStop():
- return None
-
- res = self.sf.fetchUrl(link, timeout=self.opts['_fetchtimeout'],
- useragent=self.opts['_useragent'])
-
- if res['content'] is None:
- self.sf.debug("Ignoring " + link + " as no data returned")
- continue
-
- # Sometimes pastes search results false positives
- if re.search("[^a-zA-Z\-\_0-9]" + re.escape(eventData) +
- "[^a-zA-Z\-\_0-9]", res['content'], re.IGNORECASE) is None:
- continue
+ # Sometimes pastes search results false positives
+ if re.search("[^a-zA-Z\-\_0-9]" + re.escape(eventData) +
+ "[^a-zA-Z\-\_0-9]", res['content'], re.IGNORECASE) is None:
+ continue
- try:
- startIndex = res['content'].index(eventData)
- except BaseException as e:
- self.sf.debug("String not found in pastes content.")
- continue
+ try:
+ startIndex = res['content'].index(eventData)
+ except BaseException as e:
+ self.sf.debug("String not found in pastes content.")
+ continue
- evt1 = SpiderFootEvent("LEAKSITE_URL", link, self.__name__, event)
- self.notifyListeners(evt1)
+ evt1 = SpiderFootEvent("LEAKSITE_URL", link, self.__name__, event)
+ self.notifyListeners(evt1)
- evt2 = SpiderFootEvent("LEAKSITE_CONTENT", res['content'], self.__name__, evt1)
- self.notifyListeners(evt2)
+ evt2 = SpiderFootEvent("LEAKSITE_CONTENT", res['content'], self.__name__, evt1)
+ self.notifyListeners(evt2)
# End of sfp_pastebin class