Skip to content

SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.

sfp_crossref: Ensure URL FQDN resolves; and code cleanup

Brendan Colese6fac4fdc46f
diff --git a/modules/sfp_crossref.py b/modules/sfp_crossref.py
index 36c27359..8878c8f5 100644
--- a/modules/sfp_crossref.py
+++ b/modules/sfp_crossref.py
@@ -22,23 +22,20 @@ class sfp_crossref(SpiderFootPlugin):
meta = {
'name': "Cross-Referencer",
- 'summary': "Identify whether other domains are associated ('Affiliates') of the target.",
+ 'summary': "Identify whether other domains are associated ('Affiliates') of the target by looking for links back to the target site(s).",
'flags': [""],
'useCases': ["Footprint"],
'categories': ["Crawling and Scanning"]
}
- # Default options
opts = {
'checkbase': True
}
- # Option descriptions
optdescs = {
"checkbase": "Check the base URL of the potential affiliate if no direct affiliation found?"
}
- # Internal results tracking
fetched = None
def setup(self, sfc, userOpts=dict()):
@@ -48,19 +45,20 @@ class sfp_crossref(SpiderFootPlugin):
for opt in list(userOpts.keys()):
self.opts[opt] = userOpts[opt]
- # What events is this module interested in for input
def watchedEvents(self):
- return ['LINKED_URL_EXTERNAL', 'SIMILARDOMAIN', 'CO_HOSTED_SITE', 'DARKNET_MENTION_URL']
+ return [
+ 'LINKED_URL_EXTERNAL',
+ 'SIMILARDOMAIN',
+ 'CO_HOSTED_SITE',
+ 'DARKNET_MENTION_URL'
+ ]
- # What events this module produces
- # This is to support the end user in selecting modules based on events
- # produced.
def producedEvents(self):
- return ["AFFILIATE_INTERNET_NAME", "AFFILIATE_WEB_CONTENT"]
+ return [
+ 'AFFILIATE_INTERNET_NAME',
+ 'AFFILIATE_WEB_CONTENT'
+ ]
- # Handle events sent to this module
- # In this module's case, eventData will be the URL or a domain which
- # was found in some content somewhere.
def handleEvent(self, event):
eventName = event.eventType
srcModuleName = event.module
@@ -68,41 +66,57 @@ class sfp_crossref(SpiderFootPlugin):
self.sf.debug(f"Received event, {eventName}, from {srcModuleName}")
- # The SIMILARDOMAIN and CO_HOSTED_SITE events supply domains,
- # not URLs. Assume HTTP.
+ # SIMILARDOMAIN and CO_HOSTED_SITE events are domains, not URLs.
+ # Assume HTTP.
if eventName in ['SIMILARDOMAIN', 'CO_HOSTED_SITE']:
- eventData = 'http://' + eventData.lower()
+ url = 'http://' + eventData.lower()
+ elif 'URL' in eventName:
+ url = eventData
+ else:
+ return None
+
+ fqdn = self.sf.urlFQDN(url)
# We are only interested in external sites for the crossref
- if self.getTarget().matches(self.sf.urlFQDN(eventData)):
- self.sf.debug("Ignoring " + eventData + " as not external")
+ if self.getTarget().matches(fqdn):
+ self.sf.debug(f"Ignoring {url} as not external")
return None
if eventData in self.fetched:
- self.sf.debug("Ignoring " + eventData + " as already tested")
+ self.sf.debug(f"Ignoring {url} as already tested")
return
- else:
- self.fetched[eventData] = True
- self.sf.debug("Testing for affiliation: " + eventData)
- res = self.sf.fetchUrl(eventData, timeout=self.opts['_fetchtimeout'],
- useragent=self.opts['_useragent'],
- sizeLimit=10000000,
- verify=False)
+ if not self.sf.resolveHost(fqdn):
+ self.sf.debug(f"Ignoring {url} as {fqdn} does not resolve")
+ return None
+
+ self.fetched[url] = True
+
+ self.sf.debug(f"Testing URL for affiliation: {url}")
+
+ res = self.sf.fetchUrl(
+ url,
+ timeout=self.opts['_fetchtimeout'],
+ useragent=self.opts['_useragent'],
+ sizeLimit=10000000,
+ verify=False
+ )
if res['content'] is None:
- self.sf.debug("Ignoring " + eventData + " as no data returned")
+ self.sf.debug(f"Ignoring {url} as no data returned")
return None
matched = False
for name in self.getTarget().getNames():
# Search for mentions of our host/domain in the external site's data
- pat = re.compile(r"([\.\'\/\"\ ]" + name + r"[\.\'\/\"\ ])", re.IGNORECASE)
+ pat = re.compile(
+ r"([\.\'\/\"\ ]" + re.escape(name) + r"[\.\'\/\"\ ])",
+ re.IGNORECASE
+ )
matches = re.findall(pat, res['content'])
if len(matches) > 0:
matched = True
- url = eventData
break
if not matched:
@@ -113,33 +127,53 @@ class sfp_crossref(SpiderFootPlugin):
url = self.sf.urlBaseUrl(eventData)
if url in self.fetched:
return None
- else:
- self.fetched[url] = True
- res = self.sf.fetchUrl(url, timeout=self.opts['_fetchtimeout'],
- useragent=self.opts['_useragent'],
- sizeLimit=10000000,
- verify=False)
+ self.fetched[url] = True
+
+ res = self.sf.fetchUrl(
+ url,
+ timeout=self.opts['_fetchtimeout'],
+ useragent=self.opts['_useragent'],
+ sizeLimit=10000000,
+ verify=False
+ )
+
if res['content'] is not None:
for name in self.getTarget().getNames():
- pat = re.compile(r"([\.\'\/\"\ ]" + name + r"[\'\/\"\ ])",
- re.IGNORECASE)
+ pat = re.compile(
+ r"([\.\'\/\"\ ]" + re.escape(name) + r"[\'\/\"\ ])",
+ re.IGNORECASE
+ )
matches = re.findall(pat, res['content'])
if len(matches) > 0:
matched = True
+ break
+
+ if not matched:
+ return None
- if matched:
- if not event.moduleDataSource:
- event.moduleDataSource = "Unknown"
- self.sf.info("Found affiliate: " + url)
- evt1 = SpiderFootEvent("AFFILIATE_INTERNET_NAME", self.sf.urlFQDN(url),
- self.__name__, event)
- evt1.moduleDataSource = event.moduleDataSource
- self.notifyListeners(evt1)
- evt2 = SpiderFootEvent("AFFILIATE_WEB_CONTENT", res['content'],
- self.__name__, evt1)
- evt2.moduleDataSource = event.moduleDataSource
- self.notifyListeners(evt2)
+ if not event.moduleDataSource:
+ event.moduleDataSource = "Unknown"
+
+ self.sf.info(f"Found link to target from affiliate: {url}")
+
+ evt1 = SpiderFootEvent(
+ "AFFILIATE_INTERNET_NAME",
+ self.sf.urlFQDN(url),
+ self.__name__,
+ event
+ )
+ evt1.moduleDataSource = event.moduleDataSource
+ self.notifyListeners(evt1)
+
+ evt2 = SpiderFootEvent(
+ "AFFILIATE_WEB_CONTENT",
+ res['content'],
+ self.__name__,
+ evt1
+ )
+ evt2.moduleDataSource = event.moduleDataSource
+ self.notifyListeners(evt2)
# End of sfp_crossref class