SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.
Python attacksurfacecticybersecurityfootprintinghackinginformation-gatheringinformation-securityinfosecintelligence-gatheringosintosint-frameworkosint-reconnaissanceosint-toolpentestingpythonreconsecurity-toolsthreat-intelligencethreatintel
sfp_spider: Store TARGET_WEB_CONTENT web content event as string (#1320)
bcolesd2850003d590
diff --git a/modules/sfp_spider.py b/modules/sfp_spider.pyindex 19d24e80..d4799f05 100644--- a/modules/sfp_spider.py+++ b/modules/sfp_spider.py@@ -208,45 +208,55 @@ class sfp_spider(SpiderFootPlugin):# Notify listening modules about raw data and othersdef contentNotify(self, url, httpresult, parentEvent=None):- sendcontent = True- if httpresult.get('headers'):- ctype = httpresult['headers'].get('content-type')- if not ctype:- sendcontent = True- else:- for mt in self.opts['filtermime']:- if ctype.startswith(mt):- sendcontent = False-- if sendcontent:- if httpresult['content'] is not None:- event = SpiderFootEvent("TARGET_WEB_CONTENT", httpresult['content'],- self.__name__, parentEvent)- event.actualSource = url- self.notifyListeners(event)-- hdr = httpresult['headers']- if hdr is not None:- event = SpiderFootEvent("WEBSERVER_HTTPHEADERS", json.dumps(hdr, ensure_ascii=False),- self.__name__, parentEvent)- event.actualSource = url- self.notifyListeners(event)-- event = SpiderFootEvent("HTTP_CODE", str(httpresult['code']),- self.__name__, parentEvent)+ event = SpiderFootEvent(+ "HTTP_CODE",+ str(httpresult['code']),+ self.__name__,+ parentEvent+ )event.actualSource = urlself.notifyListeners(event)- if not httpresult.get('headers'):- return+ store_content = True+ headers = httpresult.get('headers')- ctype = httpresult['headers'].get('content-type')- if ctype:- event = SpiderFootEvent("TARGET_WEB_CONTENT_TYPE", ctype,- self.__name__, parentEvent)+ if headers:+ event = SpiderFootEvent(+ "WEBSERVER_HTTPHEADERS",+ json.dumps(headers, ensure_ascii=False),+ self.__name__,+ parentEvent+ )event.actualSource = urlself.notifyListeners(event)+ ctype = headers.get('content-type')+ if ctype:+ for mt in self.opts['filtermime']:+ if ctype.startswith(mt):+ store_content = False++ event = SpiderFootEvent(+ "TARGET_WEB_CONTENT_TYPE",+ ctype,+ self.__name__,+ parentEvent+ )+ event.actualSource = url+ self.notifyListeners(event)++ if store_content:+ content = httpresult.get('content')+ if content:+ event = SpiderFootEvent(+ "TARGET_WEB_CONTENT",+ str(content),+ self.__name__,+ parentEvent+ )+ event.actualSource = url+ self.notifyListeners(event)+# Trigger spidering off the following events..# Spidering and search engines provide LINKED_URL_INTERNAL, and DNS lookups# provide INTERNET_NAME.