Skip to content

SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.

sfp_spider: Reduce complexity of spiderFrom function (#1693)

bcoles1337e1722326
diff --git a/modules/sfp_spider.py b/modules/sfp_spider.py
index 42694350..0a6be619 100644
--- a/modules/sfp_spider.py
+++ b/modules/sfp_spider.py
@@ -83,31 +83,56 @@ class sfp_spider(SpiderFootPlugin):
for opt in list(userOpts.keys()):
self.opts[opt] = userOpts[opt]
- # Fetch data from a URL and obtain all links that should be followed
- def processUrl(self, url):
+ # Search engines and DNS lookups provide INTERNET_NAME.
+ def watchedEvents(self):
+ return ["LINKED_URL_INTERNAL", "INTERNET_NAME"]
+
+ # What events this module produces
+ def producedEvents(self):
+ return [
+ "WEBSERVER_HTTPHEADERS",
+ "HTTP_CODE",
+ "LINKED_URL_INTERNAL",
+ "LINKED_URL_EXTERNAL",
+ "TARGET_WEB_CONTENT",
+ "TARGET_WEB_CONTENT_TYPE"
+ ]
+
+ def processUrl(self, url: str) -> dict:
+ """Fetch data from a URL and obtain all links that should be followed.
+
+ Args:
+ url (str): URL to fetch
+
+ Returns:
+ dict: links identified in URL content
+ """
site = self.sf.urlFQDN(url)
cookies = None
# Filter out certain file types (if user chooses to)
if list(filter(lambda ext: url.lower().split('?')[0].endswith('.' + ext.lower()), self.opts['filterfiles'])):
- # self.debug('Ignoring filtered extension: ' + link)
+ # self.debug(f"Ignoring URL with filtered file extension: {link}")
return None
if site in self.siteCookies:
self.debug(f"Restoring cookies for {site}: {self.siteCookies[site]}")
cookies = self.siteCookies[site]
- # Fetch the contents of the supplied URL (object returned)
+
+ # Fetch the contents of the supplied URL
fetched = self.sf.fetchUrl(
url,
- False,
- cookies,
- self.opts['_fetchtimeout'],
- self.opts['_useragent'],
+ cookies=cookies,
+ timeout=self.opts['_fetchtimeout'],
+ useragent=self.opts['_useragent'],
sizeLimit=10000000,
verify=False
)
self.fetchedPages[url] = True
+ if not fetched:
+ return None
+
# Track cookies a site has sent, then send the back in subsquent requests
if self.opts['usecookies'] and fetched['headers'] is not None:
if fetched['headers'].get('Set-Cookie'):
@@ -122,19 +147,27 @@ class sfp_spider(SpiderFootPlugin):
# Notify modules about the content obtained
self.contentNotify(url, fetched, self.urlEvents[url])
- if fetched['realurl'] is not None and fetched['realurl'] != url:
- # self.debug("Redirect of " + url + " to " + fetched['realurl'])
+ real_url = fetched['realurl']
+ if real_url and real_url != url:
+ # self.debug(f"Redirect of {url} to {real_url}")
# Store the content for the redirect so that it isn't fetched again
- self.fetchedPages[fetched['realurl']] = True
+ self.fetchedPages[real_url] = True
# Notify modules about the new link
- self.urlEvents[fetched['realurl']] = self.linkNotify(fetched['realurl'],
- self.urlEvents[url])
- url = fetched['realurl'] # override the URL if we had a redirect
+ self.urlEvents[real_url] = self.linkNotify(real_url, self.urlEvents[url])
+ url = real_url # override the URL if we had a redirect
+
+ data = fetched['content']
+
+ if not data:
+ return None
+
+ if isinstance(data, bytes):
+ data = data.decode('utf-8', errors='replace')
# Extract links from the content
links = SpiderFootHelpers.extractLinksFromHtml(
url,
- fetched['content'],
+ data,
self.getTarget().getNames()
)
@@ -152,11 +185,18 @@ class sfp_spider(SpiderFootPlugin):
# Supply the SpiderFootEvent of the parent URL as the parent
self.urlEvents[link] = self.linkNotify(link, self.urlEvents[url])
- self.debug('Links found from parsing: ' + str(links))
+ self.debug(f"Links found from parsing: {links.keys()}")
return links
- # Clear out links that we don't want to follow
- def cleanLinks(self, links):
+ def cleanLinks(self, links: list) -> list:
+ """Clear out links that we don't want to follow.
+
+ Args:
+ links (list): links
+
+ Returns:
+ list: links suitable for spidering
+ """
returnLinks = dict()
for link in links:
@@ -191,13 +231,13 @@ class sfp_spider(SpiderFootPlugin):
continue
# All tests passed, add link to be spidered
- self.debug("Adding URL for spidering: " + link)
+ self.debug(f"Adding URL for spidering: {link}")
returnLinks[link] = links[link]
- return returnLinks
+ return list(returnLinks.keys())
# Notify listening modules about links
- def linkNotify(self, url, parentEvent=None):
+ def linkNotify(self, url: str, parentEvent=None):
if self.getTarget().matches(self.sf.urlFQDN(url)):
utype = "LINKED_URL_INTERNAL"
else:
@@ -210,7 +250,10 @@ class sfp_spider(SpiderFootPlugin):
return event
# Notify listening modules about raw data and others
- def contentNotify(self, url, httpresult, parentEvent=None):
+ def contentNotify(self, url: str, httpresult: dict, parentEvent=None) -> None:
+ if not isinstance(httpresult, dict):
+ return
+
event = SpiderFootEvent(
"HTTP_CODE",
str(httpresult['code']),
@@ -260,19 +303,7 @@ class sfp_spider(SpiderFootPlugin):
event.actualSource = url
self.notifyListeners(event)
- # Trigger spidering off the following events..
- # Spidering and search engines provide LINKED_URL_INTERNAL, and DNS lookups
- # provide INTERNET_NAME.
- def watchedEvents(self):
- return ["LINKED_URL_INTERNAL", "INTERNET_NAME"]
-
- # What events this module produces
- def producedEvents(self):
- return ["WEBSERVER_HTTPHEADERS", "HTTP_CODE", "LINKED_URL_INTERNAL",
- "LINKED_URL_EXTERNAL", "TARGET_WEB_CONTENT", "TARGET_WEB_CONTENT_TYPE"]
-
- # Some other modules may request we spider things
- def handleEvent(self, event):
+ def handleEvent(self, event) -> None:
eventName = event.eventType
srcModuleName = event.module
eventData = event.data
@@ -286,7 +317,7 @@ class sfp_spider(SpiderFootPlugin):
return None
if eventData in self.urlEvents:
- self.debug("Ignoring " + eventData + " as already spidered or is being spidered.")
+ self.debug(f"Ignoring {eventData} as already spidered or is being spidered.")
return None
self.urlEvents[eventData] = event
@@ -294,106 +325,100 @@ class sfp_spider(SpiderFootPlugin):
# Determine where to start spidering from if it's a INTERNET_NAME event
if eventName == "INTERNET_NAME":
for prefix in self.opts['start']:
- res = self.sf.fetchUrl(prefix + eventData, timeout=self.opts['_fetchtimeout'],
- useragent=self.opts['_useragent'],
- verify=False)
+ res = self.sf.fetchUrl(
+ prefix + eventData,
+ timeout=self.opts['_fetchtimeout'],
+ useragent=self.opts['_useragent'],
+ verify=False
+ )
+
+ if not res:
+ continue
+
if res['content'] is not None:
spiderTarget = prefix + eventData
- evt = SpiderFootEvent("LINKED_URL_INTERNAL", spiderTarget,
- self.__name__, event)
+ evt = SpiderFootEvent(
+ "LINKED_URL_INTERNAL",
+ spiderTarget,
+ self.__name__,
+ event
+ )
self.notifyListeners(evt)
break
else:
spiderTarget = eventData
- if spiderTarget is None:
+ if not spiderTarget:
+ self.info(f"No reply from {eventData}, aborting.")
return None
- self.debug("Initiating spider of " + spiderTarget + " from " + srcModuleName)
+ self.debug(f"Initiating spider of {spiderTarget} from {srcModuleName}")
# Link the spidered URL to the event that triggered it
self.urlEvents[spiderTarget] = event
return self.spiderFrom(spiderTarget)
- # Start spidering
- def spiderFrom(self, startingPoint):
- keepSpidering = True
- totalFetched = 0
+ def spiderFrom(self, startingPoint: str) -> None:
+ pagesFetched = 0
levelsTraversed = 0
- nextLinks = dict()
- targetBase = SpiderFootHelpers.urlBaseUrl(startingPoint)
# Are we respecting robots.txt?
- if self.opts['robotsonly'] and targetBase not in self.robotsRules:
- robotsTxt = self.sf.fetchUrl(targetBase + '/robots.txt',
- timeout=self.opts['_fetchtimeout'],
- useragent=self.opts['_useragent'],
- verify=False)
- if robotsTxt['content'] is not None:
- self.debug('robots.txt contents: ' + robotsTxt['content'])
- self.robotsRules[targetBase] = SpiderFootHelpers.extractUrlsFromRobotsTxt(robotsTxt['content'])
-
- if self.checkForStop():
- return
+ if self.opts['robotsonly']:
+ targetBase = SpiderFootHelpers.urlBaseUrl(startingPoint)
+ if targetBase not in self.robotsRules:
+ res = self.sf.fetchUrl(
+ targetBase + '/robots.txt',
+ timeout=self.opts['_fetchtimeout'],
+ useragent=self.opts['_useragent'],
+ verify=False
+ )
+ if res:
+ robots_txt = res['content']
+ if robots_txt:
+ self.debug(f"robots.txt contents: {robots_txt}")
+ self.robotsRules[targetBase] = SpiderFootHelpers.extractUrlsFromRobotsTxt(robots_txt)
+
+ # First iteration we are starting with the target link.
+ nextLinks = [startingPoint]
+
+ # Iterations after that are based on links found on those pages, while:
+ # number of spidered pages < max pages
+ # spidering depth <= max levels (the first level is the first link)
+ while (pagesFetched < self.opts['maxpages']) and (levelsTraversed <= self.opts['maxlevels']):
+ if not nextLinks:
+ self.info("No more links to spider, finishing.")
+ return
+
+ # Fetch content from the new links
+ links = dict()
+ for link in nextLinks:
+ if self.checkForStop():
+ return
+
+ if link in self.fetchedPages:
+ self.debug(f"Already fetched {link}, skipping.")
+ continue
- # First iteration we are starting with links found on the start page
- # Iterations after that are based on links found on those pages,
- # and so on..
- links = self.processUrl(startingPoint) # fetch first page
+ self.debug(f"Fetching fresh content from: {link}")
- if links is None:
- self.debug("No links found on the first fetch!")
- return
+ time.sleep(self.opts['pausesec'])
- while keepSpidering:
- # Gets hit in the second and subsequent iterations when more links
- # are found
- if len(nextLinks) > 0:
- links = dict()
-
- # Fetch content from the new links
- for link in nextLinks:
- # Always skip links we've already fetched
- if (link in self.fetchedPages):
- self.debug("Already fetched " + link + ", skipping.")
- continue
-
- # Check if we've been asked to stop
- if self.checkForStop():
- return
-
- self.debug("Fetching fresh content from: " + link)
- time.sleep(self.opts['pausesec'])
- freshLinks = self.processUrl(link)
- if freshLinks is not None:
- links.update(freshLinks)
-
- totalFetched += 1
- if totalFetched >= self.opts['maxpages']:
- self.info("Maximum number of pages (" + str(self.opts['maxpages'])
- + ") reached.")
- keepSpidering = False
- break
+ freshLinks = self.processUrl(link)
+ if freshLinks:
+ links.update(freshLinks)
+
+ pagesFetched += 1
+ if pagesFetched >= self.opts['maxpages']:
+ self.info(f"Maximum number of pages ({self.opts['maxpages']}) reached.")
+ return
nextLinks = self.cleanLinks(links)
self.debug(f"Found links: {nextLinks}")
# We've scanned through another layer of the site
levelsTraversed += 1
- self.debug(f"At level: {levelsTraversed}, Pages: {totalFetched}")
- if levelsTraversed >= self.opts['maxlevels']:
+ self.debug(f"At level: {levelsTraversed}, Pages: {pagesFetched}")
+ if levelsTraversed > self.opts['maxlevels']:
self.info(f"Maximum number of levels ({self.opts['maxlevels']}) reached.")
- keepSpidering = False
-
- # We've reached the end of our journey..
- if len(nextLinks) == 0:
- self.debug("No more links found to spider, finishing..")
- keepSpidering = False
-
- # We've been asked to stop scanning
- if self.checkForStop():
- keepSpidering = False
-
- return
# End of sfp_spider class