SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.
Python attacksurfacecticybersecurityfootprintinghackinginformation-gatheringinformation-securityinfosecintelligence-gatheringosintosint-frameworkosint-reconnaissanceosint-toolpentestingpythonreconsecurity-toolsthreat-intelligencethreatintel
sfp_crossref: Ensure URL FQDN resolves; and code cleanup
Brendan Colese6fac4fdc46f
diff --git a/modules/sfp_crossref.py b/modules/sfp_crossref.pyindex 36c27359..8878c8f5 100644--- a/modules/sfp_crossref.py+++ b/modules/sfp_crossref.py@@ -22,23 +22,20 @@ class sfp_crossref(SpiderFootPlugin):meta = {'name': "Cross-Referencer",- 'summary': "Identify whether other domains are associated ('Affiliates') of the target.",+ 'summary': "Identify whether other domains are associated ('Affiliates') of the target by looking for links back to the target site(s).",'flags': [""],'useCases': ["Footprint"],'categories': ["Crawling and Scanning"]}- # Default optionsopts = {'checkbase': True}- # Option descriptionsoptdescs = {"checkbase": "Check the base URL of the potential affiliate if no direct affiliation found?"}- # Internal results trackingfetched = Nonedef setup(self, sfc, userOpts=dict()):@@ -48,19 +45,20 @@ class sfp_crossref(SpiderFootPlugin):for opt in list(userOpts.keys()):self.opts[opt] = userOpts[opt]- # What events is this module interested in for inputdef watchedEvents(self):- return ['LINKED_URL_EXTERNAL', 'SIMILARDOMAIN', 'CO_HOSTED_SITE', 'DARKNET_MENTION_URL']+ return [+ 'LINKED_URL_EXTERNAL',+ 'SIMILARDOMAIN',+ 'CO_HOSTED_SITE',+ 'DARKNET_MENTION_URL'+ ]- # What events this module produces- # This is to support the end user in selecting modules based on events- # produced.def producedEvents(self):- return ["AFFILIATE_INTERNET_NAME", "AFFILIATE_WEB_CONTENT"]+ return [+ 'AFFILIATE_INTERNET_NAME',+ 'AFFILIATE_WEB_CONTENT'+ ]- # Handle events sent to this module- # In this module's case, eventData will be the URL or a domain which- # was found in some content somewhere.def handleEvent(self, event):eventName = event.eventTypesrcModuleName = event.module@@ -68,41 +66,57 @@ class sfp_crossref(SpiderFootPlugin):self.sf.debug(f"Received event, {eventName}, from {srcModuleName}")- # The SIMILARDOMAIN and CO_HOSTED_SITE events supply domains,- # not URLs. Assume HTTP.+ # SIMILARDOMAIN and CO_HOSTED_SITE events are domains, not URLs.+ # Assume HTTP.if eventName in ['SIMILARDOMAIN', 'CO_HOSTED_SITE']:- eventData = 'http://' + eventData.lower()+ url = 'http://' + eventData.lower()+ elif 'URL' in eventName:+ url = eventData+ else:+ return None++ fqdn = self.sf.urlFQDN(url)# We are only interested in external sites for the crossref- if self.getTarget().matches(self.sf.urlFQDN(eventData)):- self.sf.debug("Ignoring " + eventData + " as not external")+ if self.getTarget().matches(fqdn):+ self.sf.debug(f"Ignoring {url} as not external")return Noneif eventData in self.fetched:- self.sf.debug("Ignoring " + eventData + " as already tested")+ self.sf.debug(f"Ignoring {url} as already tested")return- else:- self.fetched[eventData] = True- self.sf.debug("Testing for affiliation: " + eventData)- res = self.sf.fetchUrl(eventData, timeout=self.opts['_fetchtimeout'],- useragent=self.opts['_useragent'],- sizeLimit=10000000,- verify=False)+ if not self.sf.resolveHost(fqdn):+ self.sf.debug(f"Ignoring {url} as {fqdn} does not resolve")+ return None++ self.fetched[url] = True++ self.sf.debug(f"Testing URL for affiliation: {url}")++ res = self.sf.fetchUrl(+ url,+ timeout=self.opts['_fetchtimeout'],+ useragent=self.opts['_useragent'],+ sizeLimit=10000000,+ verify=False+ )if res['content'] is None:- self.sf.debug("Ignoring " + eventData + " as no data returned")+ self.sf.debug(f"Ignoring {url} as no data returned")return Nonematched = Falsefor name in self.getTarget().getNames():# Search for mentions of our host/domain in the external site's data- pat = re.compile(r"([\.\'\/\"\ ]" + name + r"[\.\'\/\"\ ])", re.IGNORECASE)+ pat = re.compile(+ r"([\.\'\/\"\ ]" + re.escape(name) + r"[\.\'\/\"\ ])",+ re.IGNORECASE+ )matches = re.findall(pat, res['content'])if len(matches) > 0:matched = True- url = eventDatabreakif not matched:@@ -113,33 +127,53 @@ class sfp_crossref(SpiderFootPlugin):url = self.sf.urlBaseUrl(eventData)if url in self.fetched:return None- else:- self.fetched[url] = True- res = self.sf.fetchUrl(url, timeout=self.opts['_fetchtimeout'],- useragent=self.opts['_useragent'],- sizeLimit=10000000,- verify=False)+ self.fetched[url] = True++ res = self.sf.fetchUrl(+ url,+ timeout=self.opts['_fetchtimeout'],+ useragent=self.opts['_useragent'],+ sizeLimit=10000000,+ verify=False+ )+if res['content'] is not None:for name in self.getTarget().getNames():- pat = re.compile(r"([\.\'\/\"\ ]" + name + r"[\'\/\"\ ])",- re.IGNORECASE)+ pat = re.compile(+ r"([\.\'\/\"\ ]" + re.escape(name) + r"[\'\/\"\ ])",+ re.IGNORECASE+ )matches = re.findall(pat, res['content'])if len(matches) > 0:matched = True+ break++ if not matched:+ return None- if matched:- if not event.moduleDataSource:- event.moduleDataSource = "Unknown"- self.sf.info("Found affiliate: " + url)- evt1 = SpiderFootEvent("AFFILIATE_INTERNET_NAME", self.sf.urlFQDN(url),- self.__name__, event)- evt1.moduleDataSource = event.moduleDataSource- self.notifyListeners(evt1)- evt2 = SpiderFootEvent("AFFILIATE_WEB_CONTENT", res['content'],- self.__name__, evt1)- evt2.moduleDataSource = event.moduleDataSource- self.notifyListeners(evt2)+ if not event.moduleDataSource:+ event.moduleDataSource = "Unknown"++ self.sf.info(f"Found link to target from affiliate: {url}")++ evt1 = SpiderFootEvent(+ "AFFILIATE_INTERNET_NAME",+ self.sf.urlFQDN(url),+ self.__name__,+ event+ )+ evt1.moduleDataSource = event.moduleDataSource+ self.notifyListeners(evt1)++ evt2 = SpiderFootEvent(+ "AFFILIATE_WEB_CONTENT",+ res['content'],+ self.__name__,+ evt1+ )+ evt2.moduleDataSource = event.moduleDataSource+ self.notifyListeners(evt2)# End of sfp_crossref class