Skip to content

SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.

Re-factoring due to split up of modules.

Steve0975a804414d
diff --git a/modules/sfp_pastebin.py b/modules/sfp_pastebin.py
index 2f4a49e6..8b6a1464 100644
--- a/modules/sfp_pastebin.py
+++ b/modules/sfp_pastebin.py
@@ -62,65 +62,64 @@ class sfp_pastebin(SpiderFootPlugin):
self.results.append(eventData)
for dom in self.domains.keys():
- if self.opts[dom]:
- links = list()
- target = self.domains[dom]
- pages = self.sf.googleIterate("site:" + target + "%20+\"" + eventData + "\"",
- dict(limit=self.opts['searchpages'],
- useragent=self.opts['_useragent'],
- timeout=self.opts['_fetchtimeout']))
-
- if pages is None:
- self.sf.info("No results returned from Google search of " + target + ".")
+ links = list()
+ target = self.domains[dom]
+ pages = self.sf.googleIterate("site:" + target + "%20+\"" + eventData + "\"",
+ dict(limit=self.opts['searchpages'],
+ useragent=self.opts['_useragent'],
+ timeout=self.opts['_fetchtimeout']))
+
+ if pages is None:
+ self.sf.info("No results returned from Google search of " + target + ".")
+ return None
+
+ for page in pages.keys():
+ if page in self.results:
+ continue
+ else:
+ self.results.append(page)
+
+ # Check if we've been asked to stop
+ if self.checkForStop():
return None
- for page in pages.keys():
- if page in self.results:
- continue
- else:
- self.results.append(page)
+ # Fetch the paste site content
+ links += self.sf.parseLinks(page, pages[page], target)
+
+ for link in links:
+ if link in self.results:
+ continue
+ else:
+ self.results.append(link)
- # Check if we've been asked to stop
+ self.sf.debug("Found a link: " + link)
+ if self.sf.urlBaseUrl(link).endswith(target):
if self.checkForStop():
return None
- # Fetch the paste site content
- links += self.sf.parseLinks(page, pages[page], target)
+ res = self.sf.fetchUrl(link, timeout=self.opts['_fetchtimeout'],
+ useragent=self.opts['_useragent'])
- for link in links:
- if link in self.results:
+ if res['content'] is None:
+ self.sf.debug("Ignoring " + link + " as no data returned")
continue
- else:
- self.results.append(link)
-
- self.sf.debug("Found a link: " + link)
- if self.sf.urlBaseUrl(link).endswith(target):
- if self.checkForStop():
- return None
- res = self.sf.fetchUrl(link, timeout=self.opts['_fetchtimeout'],
- useragent=self.opts['_useragent'])
+ # Sometimes pastes search results false positives
+ if re.search("[^a-zA-Z\-\_0-9]" + re.escape(eventData) +
+ "[^a-zA-Z\-\_0-9]", res['content'], re.IGNORECASE) is None:
+ continue
- if res['content'] is None:
- self.sf.debug("Ignoring " + link + " as no data returned")
- continue
-
- # Sometimes pastes search results false positives
- if re.search("[^a-zA-Z\-\_0-9]" + re.escape(eventData) +
- "[^a-zA-Z\-\_0-9]", res['content'], re.IGNORECASE) is None:
- continue
-
- try:
- startIndex = res['content'].index(eventData)
- except BaseException as e:
- self.sf.debug("String not found in pastes content.")
- continue
+ try:
+ startIndex = res['content'].index(eventData)
+ except BaseException as e:
+ self.sf.debug("String not found in pastes content.")
+ continue
- evt1 = SpiderFootEvent("LEAKSITE_URL", link, self.__name__, event)
- self.notifyListeners(evt1)
+ evt1 = SpiderFootEvent("LEAKSITE_URL", link, self.__name__, event)
+ self.notifyListeners(evt1)
- evt2 = SpiderFootEvent("LEAKSITE_CONTENT", res['content'], self.__name__, evt1)
- self.notifyListeners(evt2)
+ evt2 = SpiderFootEvent("LEAKSITE_CONTENT", res['content'], self.__name__, evt1)
+ self.notifyListeners(evt2)
# End of sfp_pastebin class
diff --git a/modules/sfp_pastie.py b/modules/sfp_pastie.py
index 0dac6783..53ae067d 100644
--- a/modules/sfp_pastie.py
+++ b/modules/sfp_pastie.py
@@ -62,65 +62,64 @@ class sfp_pastie(SpiderFootPlugin):
self.results.append(eventData)
for dom in self.domains.keys():
- if self.opts[dom]:
- links = list()
- target = self.domains[dom]
- pages = self.sf.googleIterate("site:" + target + "%20+\"" + eventData + "\"",
- dict(limit=self.opts['searchpages'],
- useragent=self.opts['_useragent'],
- timeout=self.opts['_fetchtimeout']))
-
- if pages is None:
- self.sf.info("No results returned from Google search of " + target + ".")
+ links = list()
+ target = self.domains[dom]
+ pages = self.sf.googleIterate("site:" + target + "%20+\"" + eventData + "\"",
+ dict(limit=self.opts['searchpages'],
+ useragent=self.opts['_useragent'],
+ timeout=self.opts['_fetchtimeout']))
+
+ if pages is None:
+ self.sf.info("No results returned from Google search of " + target + ".")
+ return None
+
+ for page in pages.keys():
+ if page in self.results:
+ continue
+ else:
+ self.results.append(page)
+
+ # Check if we've been asked to stop
+ if self.checkForStop():
return None
- for page in pages.keys():
- if page in self.results:
- continue
- else:
- self.results.append(page)
+ # Fetch the paste site content
+ links += self.sf.parseLinks(page, pages[page], target)
+
+ for link in links:
+ if link in self.results:
+ continue
+ else:
+ self.results.append(link)
- # Check if we've been asked to stop
+ self.sf.debug("Found a link: " + link)
+ if self.sf.urlBaseUrl(link).endswith(target):
if self.checkForStop():
return None
- # Fetch the paste site content
- links += self.sf.parseLinks(page, pages[page], target)
+ res = self.sf.fetchUrl(link, timeout=self.opts['_fetchtimeout'],
+ useragent=self.opts['_useragent'])
- for link in links:
- if link in self.results:
+ if res['content'] is None:
+ self.sf.debug("Ignoring " + link + " as no data returned")
continue
- else:
- self.results.append(link)
-
- self.sf.debug("Found a link: " + link)
- if self.sf.urlBaseUrl(link).endswith(target):
- if self.checkForStop():
- return None
- res = self.sf.fetchUrl(link, timeout=self.opts['_fetchtimeout'],
- useragent=self.opts['_useragent'])
+ # Sometimes pastes search results false positives
+ if re.search("[^a-zA-Z\-\_0-9]" + re.escape(eventData) +
+ "[^a-zA-Z\-\_0-9]", res['content'], re.IGNORECASE) is None:
+ continue
- if res['content'] is None:
- self.sf.debug("Ignoring " + link + " as no data returned")
- continue
-
- # Sometimes pastes search results false positives
- if re.search("[^a-zA-Z\-\_0-9]" + re.escape(eventData) +
- "[^a-zA-Z\-\_0-9]", res['content'], re.IGNORECASE) is None:
- continue
-
- try:
- startIndex = res['content'].index(eventData)
- except BaseException as e:
- self.sf.debug("String not found in pastes content.")
- continue
+ try:
+ startIndex = res['content'].index(eventData)
+ except BaseException as e:
+ self.sf.debug("String not found in pastes content.")
+ continue
- evt1 = SpiderFootEvent("LEAKSITE_URL", link, self.__name__, event)
- self.notifyListeners(evt1)
+ evt1 = SpiderFootEvent("LEAKSITE_URL", link, self.__name__, event)
+ self.notifyListeners(evt1)
- evt2 = SpiderFootEvent("LEAKSITE_CONTENT", res['content'], self.__name__, evt1)
- self.notifyListeners(evt2)
+ evt2 = SpiderFootEvent("LEAKSITE_CONTENT", res['content'], self.__name__, evt1)
+ self.notifyListeners(evt2)
# End of sfp_pastie class