SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.
Python attacksurfacecticybersecurityfootprintinghackinginformation-gatheringinformation-securityinfosecintelligence-gatheringosintosint-frameworkosint-reconnaissanceosint-toolpentestingpythonreconsecurity-toolsthreat-intelligencethreatintel
sfp_spider: Reduce complexity of spiderFrom function (#1693)
bcoles1337e1722326
diff --git a/modules/sfp_spider.py b/modules/sfp_spider.pyindex 42694350..0a6be619 100644--- a/modules/sfp_spider.py+++ b/modules/sfp_spider.py@@ -83,31 +83,56 @@ class sfp_spider(SpiderFootPlugin):for opt in list(userOpts.keys()):self.opts[opt] = userOpts[opt]- # Fetch data from a URL and obtain all links that should be followed- def processUrl(self, url):+ # Search engines and DNS lookups provide INTERNET_NAME.+ def watchedEvents(self):+ return ["LINKED_URL_INTERNAL", "INTERNET_NAME"]++ # What events this module produces+ def producedEvents(self):+ return [+ "WEBSERVER_HTTPHEADERS",+ "HTTP_CODE",+ "LINKED_URL_INTERNAL",+ "LINKED_URL_EXTERNAL",+ "TARGET_WEB_CONTENT",+ "TARGET_WEB_CONTENT_TYPE"+ ]++ def processUrl(self, url: str) -> dict:+ """Fetch data from a URL and obtain all links that should be followed.++ Args:+ url (str): URL to fetch++ Returns:+ dict: links identified in URL content+ """site = self.sf.urlFQDN(url)cookies = None# Filter out certain file types (if user chooses to)if list(filter(lambda ext: url.lower().split('?')[0].endswith('.' + ext.lower()), self.opts['filterfiles'])):- # self.debug('Ignoring filtered extension: ' + link)+ # self.debug(f"Ignoring URL with filtered file extension: {link}")return Noneif site in self.siteCookies:self.debug(f"Restoring cookies for {site}: {self.siteCookies[site]}")cookies = self.siteCookies[site]- # Fetch the contents of the supplied URL (object returned)++ # Fetch the contents of the supplied URLfetched = self.sf.fetchUrl(url,- False,- cookies,- self.opts['_fetchtimeout'],- self.opts['_useragent'],+ cookies=cookies,+ timeout=self.opts['_fetchtimeout'],+ useragent=self.opts['_useragent'],sizeLimit=10000000,verify=False)self.fetchedPages[url] = True+ if not fetched:+ return None+# Track cookies a site has sent, then send the back in subsquent requestsif self.opts['usecookies'] and fetched['headers'] is not None:if fetched['headers'].get('Set-Cookie'):@@ -122,19 +147,27 @@ class sfp_spider(SpiderFootPlugin):# Notify modules about the content obtainedself.contentNotify(url, fetched, self.urlEvents[url])- if fetched['realurl'] is not None and fetched['realurl'] != url:- # self.debug("Redirect of " + url + " to " + fetched['realurl'])+ real_url = fetched['realurl']+ if real_url and real_url != url:+ # self.debug(f"Redirect of {url} to {real_url}")# Store the content for the redirect so that it isn't fetched again- self.fetchedPages[fetched['realurl']] = True+ self.fetchedPages[real_url] = True# Notify modules about the new link- self.urlEvents[fetched['realurl']] = self.linkNotify(fetched['realurl'],- self.urlEvents[url])- url = fetched['realurl'] # override the URL if we had a redirect+ self.urlEvents[real_url] = self.linkNotify(real_url, self.urlEvents[url])+ url = real_url # override the URL if we had a redirect++ data = fetched['content']++ if not data:+ return None++ if isinstance(data, bytes):+ data = data.decode('utf-8', errors='replace')# Extract links from the contentlinks = SpiderFootHelpers.extractLinksFromHtml(url,- fetched['content'],+ data,self.getTarget().getNames())@@ -152,11 +185,18 @@ class sfp_spider(SpiderFootPlugin):# Supply the SpiderFootEvent of the parent URL as the parentself.urlEvents[link] = self.linkNotify(link, self.urlEvents[url])- self.debug('Links found from parsing: ' + str(links))+ self.debug(f"Links found from parsing: {links.keys()}")return links- # Clear out links that we don't want to follow- def cleanLinks(self, links):+ def cleanLinks(self, links: list) -> list:+ """Clear out links that we don't want to follow.++ Args:+ links (list): links++ Returns:+ list: links suitable for spidering+ """returnLinks = dict()for link in links:@@ -191,13 +231,13 @@ class sfp_spider(SpiderFootPlugin):continue# All tests passed, add link to be spidered- self.debug("Adding URL for spidering: " + link)+ self.debug(f"Adding URL for spidering: {link}")returnLinks[link] = links[link]- return returnLinks+ return list(returnLinks.keys())# Notify listening modules about links- def linkNotify(self, url, parentEvent=None):+ def linkNotify(self, url: str, parentEvent=None):if self.getTarget().matches(self.sf.urlFQDN(url)):utype = "LINKED_URL_INTERNAL"else:@@ -210,7 +250,10 @@ class sfp_spider(SpiderFootPlugin):return event# Notify listening modules about raw data and others- def contentNotify(self, url, httpresult, parentEvent=None):+ def contentNotify(self, url: str, httpresult: dict, parentEvent=None) -> None:+ if not isinstance(httpresult, dict):+ return+event = SpiderFootEvent("HTTP_CODE",str(httpresult['code']),@@ -260,19 +303,7 @@ class sfp_spider(SpiderFootPlugin):event.actualSource = urlself.notifyListeners(event)- # Trigger spidering off the following events..- # Spidering and search engines provide LINKED_URL_INTERNAL, and DNS lookups- # provide INTERNET_NAME.- def watchedEvents(self):- return ["LINKED_URL_INTERNAL", "INTERNET_NAME"]-- # What events this module produces- def producedEvents(self):- return ["WEBSERVER_HTTPHEADERS", "HTTP_CODE", "LINKED_URL_INTERNAL",- "LINKED_URL_EXTERNAL", "TARGET_WEB_CONTENT", "TARGET_WEB_CONTENT_TYPE"]-- # Some other modules may request we spider things- def handleEvent(self, event):+ def handleEvent(self, event) -> None:eventName = event.eventTypesrcModuleName = event.moduleeventData = event.data@@ -286,7 +317,7 @@ class sfp_spider(SpiderFootPlugin):return Noneif eventData in self.urlEvents:- self.debug("Ignoring " + eventData + " as already spidered or is being spidered.")+ self.debug(f"Ignoring {eventData} as already spidered or is being spidered.")return Noneself.urlEvents[eventData] = event@@ -294,106 +325,100 @@ class sfp_spider(SpiderFootPlugin):# Determine where to start spidering from if it's a INTERNET_NAME eventif eventName == "INTERNET_NAME":for prefix in self.opts['start']:- res = self.sf.fetchUrl(prefix + eventData, timeout=self.opts['_fetchtimeout'],- useragent=self.opts['_useragent'],- verify=False)+ res = self.sf.fetchUrl(+ prefix + eventData,+ timeout=self.opts['_fetchtimeout'],+ useragent=self.opts['_useragent'],+ verify=False+ )++ if not res:+ continue+if res['content'] is not None:spiderTarget = prefix + eventData- evt = SpiderFootEvent("LINKED_URL_INTERNAL", spiderTarget,- self.__name__, event)+ evt = SpiderFootEvent(+ "LINKED_URL_INTERNAL",+ spiderTarget,+ self.__name__,+ event+ )self.notifyListeners(evt)breakelse:spiderTarget = eventData- if spiderTarget is None:+ if not spiderTarget:+ self.info(f"No reply from {eventData}, aborting.")return None- self.debug("Initiating spider of " + spiderTarget + " from " + srcModuleName)+ self.debug(f"Initiating spider of {spiderTarget} from {srcModuleName}")# Link the spidered URL to the event that triggered itself.urlEvents[spiderTarget] = eventreturn self.spiderFrom(spiderTarget)- # Start spidering- def spiderFrom(self, startingPoint):- keepSpidering = True- totalFetched = 0+ def spiderFrom(self, startingPoint: str) -> None:+ pagesFetched = 0levelsTraversed = 0- nextLinks = dict()- targetBase = SpiderFootHelpers.urlBaseUrl(startingPoint)# Are we respecting robots.txt?- if self.opts['robotsonly'] and targetBase not in self.robotsRules:- robotsTxt = self.sf.fetchUrl(targetBase + '/robots.txt',- timeout=self.opts['_fetchtimeout'],- useragent=self.opts['_useragent'],- verify=False)- if robotsTxt['content'] is not None:- self.debug('robots.txt contents: ' + robotsTxt['content'])- self.robotsRules[targetBase] = SpiderFootHelpers.extractUrlsFromRobotsTxt(robotsTxt['content'])-- if self.checkForStop():- return+ if self.opts['robotsonly']:+ targetBase = SpiderFootHelpers.urlBaseUrl(startingPoint)+ if targetBase not in self.robotsRules:+ res = self.sf.fetchUrl(+ targetBase + '/robots.txt',+ timeout=self.opts['_fetchtimeout'],+ useragent=self.opts['_useragent'],+ verify=False+ )+ if res:+ robots_txt = res['content']+ if robots_txt:+ self.debug(f"robots.txt contents: {robots_txt}")+ self.robotsRules[targetBase] = SpiderFootHelpers.extractUrlsFromRobotsTxt(robots_txt)++ # First iteration we are starting with the target link.+ nextLinks = [startingPoint]++ # Iterations after that are based on links found on those pages, while:+ # number of spidered pages < max pages+ # spidering depth <= max levels (the first level is the first link)+ while (pagesFetched < self.opts['maxpages']) and (levelsTraversed <= self.opts['maxlevels']):+ if not nextLinks:+ self.info("No more links to spider, finishing.")+ return++ # Fetch content from the new links+ links = dict()+ for link in nextLinks:+ if self.checkForStop():+ return++ if link in self.fetchedPages:+ self.debug(f"Already fetched {link}, skipping.")+ continue- # First iteration we are starting with links found on the start page- # Iterations after that are based on links found on those pages,- # and so on..- links = self.processUrl(startingPoint) # fetch first page+ self.debug(f"Fetching fresh content from: {link}")- if links is None:- self.debug("No links found on the first fetch!")- return+ time.sleep(self.opts['pausesec'])- while keepSpidering:- # Gets hit in the second and subsequent iterations when more links- # are found- if len(nextLinks) > 0:- links = dict()-- # Fetch content from the new links- for link in nextLinks:- # Always skip links we've already fetched- if (link in self.fetchedPages):- self.debug("Already fetched " + link + ", skipping.")- continue-- # Check if we've been asked to stop- if self.checkForStop():- return-- self.debug("Fetching fresh content from: " + link)- time.sleep(self.opts['pausesec'])- freshLinks = self.processUrl(link)- if freshLinks is not None:- links.update(freshLinks)-- totalFetched += 1- if totalFetched >= self.opts['maxpages']:- self.info("Maximum number of pages (" + str(self.opts['maxpages'])- + ") reached.")- keepSpidering = False- break+ freshLinks = self.processUrl(link)+ if freshLinks:+ links.update(freshLinks)++ pagesFetched += 1+ if pagesFetched >= self.opts['maxpages']:+ self.info(f"Maximum number of pages ({self.opts['maxpages']}) reached.")+ returnnextLinks = self.cleanLinks(links)self.debug(f"Found links: {nextLinks}")# We've scanned through another layer of the sitelevelsTraversed += 1- self.debug(f"At level: {levelsTraversed}, Pages: {totalFetched}")- if levelsTraversed >= self.opts['maxlevels']:+ self.debug(f"At level: {levelsTraversed}, Pages: {pagesFetched}")+ if levelsTraversed > self.opts['maxlevels']:self.info(f"Maximum number of levels ({self.opts['maxlevels']}) reached.")- keepSpidering = False-- # We've reached the end of our journey..- if len(nextLinks) == 0:- self.debug("No more links found to spider, finishing..")- keepSpidering = False-- # We've been asked to stop scanning- if self.checkForStop():- keepSpidering = False-- return# End of sfp_spider class