SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.
Python attacksurfacecticybersecurityfootprintinghackinginformation-gatheringinformation-securityinfosecintelligence-gatheringosintosint-frameworkosint-reconnaissanceosint-toolpentestingpythonreconsecurity-toolsthreat-intelligencethreatintel
Add URL extraction for Location and CSP headers
Brendan Coles03301129ea89
diff --git a/modules/sfp_websvr.py b/modules/sfp_websvr.pyindex acabc7e6..cb635a15 100644--- a/modules/sfp_websvr.py+++ b/modules/sfp_websvr.py@@ -39,7 +39,8 @@ class sfp_websvr(SpiderFootPlugin):# This is to support the end user in selecting modules based on events# produced.def producedEvents(self):- return ["WEBSERVER_BANNER", "WEBSERVER_TECHNOLOGY"]+ return ["WEBSERVER_BANNER", "WEBSERVER_TECHNOLOGY",+ 'LINKED_URL_INTERNAL', 'LINKED_URL_EXTERNAL']# Handle events sent to this moduledef handleEvent(self, event):@@ -67,6 +68,28 @@ class sfp_websvr(SpiderFootPlugin):self.sf.error("Received HTTP headers from another module in an unexpected format.", False)return None+ # Check location header for linked URLs+ if 'location' in jdata:+ if jdata['location'].startswith('http://') or jdata['location'].startswith('https://'):+ if self.getTarget().matches(self.sf.urlFQDN(jdata['location'])):+ evt = SpiderFootEvent('LINKED_URL_INTERNAL', jdata['location'], self.__name__, parentEvent)+ self.notifyListeners(evt)+ else:+ evt = SpiderFootEvent('LINKED_URL_EXTERNAL', jdata['location'], self.__name__, parentEvent)+ self.notifyListeners(evt)++ # Check CSP header for linked URLs+ if 'content-security-policy' in jdata:+ for directive in jdata['content-security-policy'].split(';'):+ for string in directive.split(' '):+ if string.startswith('http://') or string.startswith('https://'):+ if self.getTarget().matches(self.sf.urlFQDN(string)):+ evt = SpiderFootEvent('LINKED_URL_INTERNAL', string, self.__name__, parentEvent)+ self.notifyListeners(evt)+ else:+ evt = SpiderFootEvent('LINKED_URL_EXTERNAL', string, self.__name__, parentEvent)+ self.notifyListeners(evt)+# Could apply some smarts here, for instance looking for certain# banners and therefore classifying them further (type and version,# possibly OS. This could also trigger additional tests, such as 404s