SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.
Python attacksurfacecticybersecurityfootprintinghackinginformation-gatheringinformation-securityinfosecintelligence-gatheringosintosint-frameworkosint-reconnaissanceosint-toolpentestingpythonreconsecurity-toolsthreat-intelligencethreatintel
Replace cgi.escape with html.escape for Python 3.8.
smicalleffd5690571fbb
diff --git a/modules/sfp__stor_stdout.py b/modules/sfp__stor_stdout.pyindex b3443a95..77323048 100644--- a/modules/sfp__stor_stdout.py+++ b/modules/sfp__stor_stdout.py@@ -17,6 +17,7 @@ import jsonclass sfp__stor_stdout(SpiderFootPlugin):"""Command-line output::::Dumps output to standard out. Used for when a SpiderFoot scan is run via the command-line."""_priority = 0+ firstEvent = True# Default optionsopts = {@@ -90,7 +91,11 @@ class sfp__stor_stdout(SpiderFootPlugin):if self.opts['_format'] == "json":d = event.asDict()d['type'] = self.opts['_eventtypes'][event.eventType]- print(json.dumps(d))+ if self.firstEvent:+ self.firstEvent = False+ else:+ print(",")+ print(json.dumps(d), end='')# Handle events sent to this moduledef handleEvent(self, sfEvent):diff --git a/modules/sfp_crt.py b/modules/sfp_crt.pyindex 6a5a0384..e40a1481 100644--- a/modules/sfp_crt.py+++ b/modules/sfp_crt.py@@ -68,7 +68,7 @@ class sfp_crt(SpiderFootPlugin):}res = self.sf.fetchUrl('https://crt.sh/?' + urllib.parse.urlencode(params),- timeout=self.opts['_fetchtimeout'],+ timeout=30,useragent=self.opts['_useragent'])if res['content'] is None:@@ -144,7 +144,7 @@ class sfp_crt(SpiderFootPlugin):}res = self.sf.fetchUrl('https://crt.sh/?' + urllib.parse.urlencode(params),- timeout=self.opts['_fetchtimeout'],+ timeout=30,useragent=self.opts['_useragent'])try:diff --git a/sf.py b/sf.pyindex 0e101829..bacaacd3 100755--- a/sf.py+++ b/sf.py@@ -103,14 +103,14 @@ if __name__ == '__main__':p.add_argument("-s", metavar="TARGET", help="Target for the scan.")p.add_argument("-t", metavar="type1,type2,...", type=str, help="Event types to collect.")p.add_argument("-T", "--types", action='store_true', help="List available event types.")- p.add_argument("-o", metavar="tab|csv|json", type=str, help="Output format. Tab is default.")+ p.add_argument("-o", metavar="tab|csv|json", type=str, help="Output format. Tab is default. If using json, -q is enforced.")p.add_argument("-H", action='store_true', help="Don't print field headers, just data.")p.add_argument("-n", action='store_true', help="Strip newlines from data.")p.add_argument("-r", action='store_true', help="Include the source data field in tab/csv output.")p.add_argument("-S", metavar="LENGTH", type=int, help="Maximum data length to display. By default, all data is shown.")p.add_argument("-D", metavar='DELIMITER', type=str, help="Delimiter to use for CSV output. Default is ,.")p.add_argument("-f", action='store_true', help="Filter out other event types that weren't requested with -t.")- p.add_argument("-F", metavar="FILTER", type=str, help="Filter out a set of event types.")+ p.add_argument("-F", metavar="FILTER", type=str, help="Show only a set of event types, comma-separated.")p.add_argument("-x", action='store_true', help="STRICT MODE. Will only enable modules that can directly consume your target, and if -t was specified only those events will be consumed by modules. This overrides -t and -m options.")p.add_argument("-q", action='store_true', help="Disable logging. This will also hide errors!")args = p.parse_args()@@ -120,7 +120,7 @@ if __name__ == '__main__':else:sfConfig['_debug'] = False- if args.q:+ if args.q or args.o == "json":sfConfig['__logging'] = Falseif args.l:@@ -306,7 +306,7 @@ if __name__ == '__main__':modlist += ["sfp__stor_db", "sfp__stor_stdout"]# Run the scan- if not args.q:+ if sfConfig['__logging']:print(("[*] Modules enabled (" + str(len(modlist)) + "): " + ",".join(modlist)))cfg = sf.configUnserialize(dbh.configGet(), sfConfig)scanId = sf.genScanInstanceGUID(target)@@ -348,17 +348,21 @@ if __name__ == '__main__':else:print('{0:30}{1}{2:45}{3}{4}{5}{6}'.format("Source", delim, "Type", delim, "Source Data", delim, "Data"))+ if args.o == "json":+ print("[", end='')+while True:info = dbh.scanInstanceGet(scanId)if not info:time.sleep(1)continueif info[5] in [ "ERROR-FAILED", "ABORT-REQUESTED", "ABORTED", "FINISHED" ]:- if not args.q:+ if sfConfig['__logging']:print("[*] Scan completed with status " + info[5])+ if args.o == "json":+ print("]")sys.exit(0)time.sleep(1)-sys.exit(0)# Start the web server so you can start looking at resultsdiff --git a/sfwebui.py b/sfwebui.pyindex 668f3631..d90f7e7f 100644--- a/sfwebui.py+++ b/sfwebui.py@@ -11,7 +11,7 @@# -----------------------------------------------------------------import jsonimport cherrypy-import cgi+import htmlimport csvimport timeimport random@@ -88,7 +88,7 @@ class SpiderFootWebUi:ret = list()for item in inputList:- c = cgi.escape(item, True)+ c = html.escape(item, True)c = c.replace('\'', '"')# We don't actually want & translated to &c = c.replace("&", "&").replace(""", "\"")@@ -123,8 +123,8 @@ class SpiderFootWebUi:retdata = []for row in data:lastseen = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(row[0]))- escapeddata = cgi.escape(row[1])- escapedsrc = cgi.escape(row[2])+ escapeddata = html.escape(row[1])+ escapedsrc = html.escape(row[2])retdata.append([lastseen, escapeddata, escapedsrc,row[3], row[5], row[6], row[7], row[8], row[10],row[11], row[4], row[13], row[14]])@@ -466,7 +466,7 @@ class SpiderFootWebUi:return self.error("Scan ID not found.")templ = Template(filename='dyn/scaninfo.tmpl', lookup=self.lookup, input_encoding='utf-8')- return templ.render(id=id, name=cgi.escape(res[0]), status=res[5], docroot=self.docroot,+ return templ.render(id=id, name=html.escape(res[0]), status=res[5], docroot=self.docroot,pageid="SCANLIST")scaninfo.exposed = True@@ -937,7 +937,7 @@ class SpiderFootWebUi:retdata = []for row in data:generated = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(row[0] / 1000))- retdata.append([generated, row[1], row[2], cgi.escape(row[3]), row[4]])+ retdata.append([generated, row[1], row[2], html.escape(row[3]), row[4]])return json.dumps(retdata)scanlog.exposed = True@@ -950,7 +950,7 @@ class SpiderFootWebUi:for row in data:generated = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(row[0] / 1000))retdata.append([generated, row[1],- cgi.escape(str(row[2]))])+ html.escape(str(row[2]))])return json.dumps(retdata)scanerrors.exposed = True@@ -1010,8 +1010,8 @@ class SpiderFootWebUi:retdata = []for row in data:lastseen = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(row[0]))- escapeddata = cgi.escape(row[1])- escapedsrc = cgi.escape(row[2])+ escapeddata = html.escape(row[1])+ escapedsrc = html.escape(row[2])retdata.append([lastseen, escapeddata, escapedsrc,row[3], row[5], row[6], row[7], row[8],row[13], row[14], row[4]])@@ -1025,7 +1025,7 @@ class SpiderFootWebUi:data = dbh.scanResultEventUnique(id, eventType, filterfp)retdata = []for row in data:- escaped = cgi.escape(row[0])+ escaped = html.escape(row[0])retdata.append([escaped, row[1], row[2]])return json.dumps(retdata, ensure_ascii=False)