Skip to content

SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.

Replace cgi.escape with html.escape for Python 3.8.

smicalleffd5690571fbb
diff --git a/modules/sfp__stor_stdout.py b/modules/sfp__stor_stdout.py
index b3443a95..77323048 100644
--- a/modules/sfp__stor_stdout.py
+++ b/modules/sfp__stor_stdout.py
@@ -17,6 +17,7 @@ import json
class sfp__stor_stdout(SpiderFootPlugin):
"""Command-line output::::Dumps output to standard out. Used for when a SpiderFoot scan is run via the command-line."""
_priority = 0
+ firstEvent = True
# Default options
opts = {
@@ -90,7 +91,11 @@ class sfp__stor_stdout(SpiderFootPlugin):
if self.opts['_format'] == "json":
d = event.asDict()
d['type'] = self.opts['_eventtypes'][event.eventType]
- print(json.dumps(d))
+ if self.firstEvent:
+ self.firstEvent = False
+ else:
+ print(",")
+ print(json.dumps(d), end='')
# Handle events sent to this module
def handleEvent(self, sfEvent):
diff --git a/modules/sfp_crt.py b/modules/sfp_crt.py
index 6a5a0384..e40a1481 100644
--- a/modules/sfp_crt.py
+++ b/modules/sfp_crt.py
@@ -68,7 +68,7 @@ class sfp_crt(SpiderFootPlugin):
}
res = self.sf.fetchUrl('https://crt.sh/?' + urllib.parse.urlencode(params),
- timeout=self.opts['_fetchtimeout'],
+ timeout=30,
useragent=self.opts['_useragent'])
if res['content'] is None:
@@ -144,7 +144,7 @@ class sfp_crt(SpiderFootPlugin):
}
res = self.sf.fetchUrl('https://crt.sh/?' + urllib.parse.urlencode(params),
- timeout=self.opts['_fetchtimeout'],
+ timeout=30,
useragent=self.opts['_useragent'])
try:
diff --git a/sf.py b/sf.py
index 0e101829..bacaacd3 100755
--- a/sf.py
+++ b/sf.py
@@ -103,14 +103,14 @@ if __name__ == '__main__':
p.add_argument("-s", metavar="TARGET", help="Target for the scan.")
p.add_argument("-t", metavar="type1,type2,...", type=str, help="Event types to collect.")
p.add_argument("-T", "--types", action='store_true', help="List available event types.")
- p.add_argument("-o", metavar="tab|csv|json", type=str, help="Output format. Tab is default.")
+ p.add_argument("-o", metavar="tab|csv|json", type=str, help="Output format. Tab is default. If using json, -q is enforced.")
p.add_argument("-H", action='store_true', help="Don't print field headers, just data.")
p.add_argument("-n", action='store_true', help="Strip newlines from data.")
p.add_argument("-r", action='store_true', help="Include the source data field in tab/csv output.")
p.add_argument("-S", metavar="LENGTH", type=int, help="Maximum data length to display. By default, all data is shown.")
p.add_argument("-D", metavar='DELIMITER', type=str, help="Delimiter to use for CSV output. Default is ,.")
p.add_argument("-f", action='store_true', help="Filter out other event types that weren't requested with -t.")
- p.add_argument("-F", metavar="FILTER", type=str, help="Filter out a set of event types.")
+ p.add_argument("-F", metavar="FILTER", type=str, help="Show only a set of event types, comma-separated.")
p.add_argument("-x", action='store_true', help="STRICT MODE. Will only enable modules that can directly consume your target, and if -t was specified only those events will be consumed by modules. This overrides -t and -m options.")
p.add_argument("-q", action='store_true', help="Disable logging. This will also hide errors!")
args = p.parse_args()
@@ -120,7 +120,7 @@ if __name__ == '__main__':
else:
sfConfig['_debug'] = False
- if args.q:
+ if args.q or args.o == "json":
sfConfig['__logging'] = False
if args.l:
@@ -306,7 +306,7 @@ if __name__ == '__main__':
modlist += ["sfp__stor_db", "sfp__stor_stdout"]
# Run the scan
- if not args.q:
+ if sfConfig['__logging']:
print(("[*] Modules enabled (" + str(len(modlist)) + "): " + ",".join(modlist)))
cfg = sf.configUnserialize(dbh.configGet(), sfConfig)
scanId = sf.genScanInstanceGUID(target)
@@ -348,17 +348,21 @@ if __name__ == '__main__':
else:
print('{0:30}{1}{2:45}{3}{4}{5}{6}'.format("Source", delim, "Type", delim, "Source Data", delim, "Data"))
+ if args.o == "json":
+ print("[", end='')
+
while True:
info = dbh.scanInstanceGet(scanId)
if not info:
time.sleep(1)
continue
if info[5] in [ "ERROR-FAILED", "ABORT-REQUESTED", "ABORTED", "FINISHED" ]:
- if not args.q:
+ if sfConfig['__logging']:
print("[*] Scan completed with status " + info[5])
+ if args.o == "json":
+ print("]")
sys.exit(0)
time.sleep(1)
-
sys.exit(0)
# Start the web server so you can start looking at results
diff --git a/sfwebui.py b/sfwebui.py
index 668f3631..d90f7e7f 100644
--- a/sfwebui.py
+++ b/sfwebui.py
@@ -11,7 +11,7 @@
# -----------------------------------------------------------------
import json
import cherrypy
-import cgi
+import html
import csv
import time
import random
@@ -88,7 +88,7 @@ class SpiderFootWebUi:
ret = list()
for item in inputList:
- c = cgi.escape(item, True)
+ c = html.escape(item, True)
c = c.replace('\'', '"')
# We don't actually want & translated to &
c = c.replace("&", "&").replace(""", "\"")
@@ -123,8 +123,8 @@ class SpiderFootWebUi:
retdata = []
for row in data:
lastseen = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(row[0]))
- escapeddata = cgi.escape(row[1])
- escapedsrc = cgi.escape(row[2])
+ escapeddata = html.escape(row[1])
+ escapedsrc = html.escape(row[2])
retdata.append([lastseen, escapeddata, escapedsrc,
row[3], row[5], row[6], row[7], row[8], row[10],
row[11], row[4], row[13], row[14]])
@@ -466,7 +466,7 @@ class SpiderFootWebUi:
return self.error("Scan ID not found.")
templ = Template(filename='dyn/scaninfo.tmpl', lookup=self.lookup, input_encoding='utf-8')
- return templ.render(id=id, name=cgi.escape(res[0]), status=res[5], docroot=self.docroot,
+ return templ.render(id=id, name=html.escape(res[0]), status=res[5], docroot=self.docroot,
pageid="SCANLIST")
scaninfo.exposed = True
@@ -937,7 +937,7 @@ class SpiderFootWebUi:
retdata = []
for row in data:
generated = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(row[0] / 1000))
- retdata.append([generated, row[1], row[2], cgi.escape(row[3]), row[4]])
+ retdata.append([generated, row[1], row[2], html.escape(row[3]), row[4]])
return json.dumps(retdata)
scanlog.exposed = True
@@ -950,7 +950,7 @@ class SpiderFootWebUi:
for row in data:
generated = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(row[0] / 1000))
retdata.append([generated, row[1],
- cgi.escape(str(row[2]))])
+ html.escape(str(row[2]))])
return json.dumps(retdata)
scanerrors.exposed = True
@@ -1010,8 +1010,8 @@ class SpiderFootWebUi:
retdata = []
for row in data:
lastseen = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(row[0]))
- escapeddata = cgi.escape(row[1])
- escapedsrc = cgi.escape(row[2])
+ escapeddata = html.escape(row[1])
+ escapedsrc = html.escape(row[2])
retdata.append([lastseen, escapeddata, escapedsrc,
row[3], row[5], row[6], row[7], row[8],
row[13], row[14], row[4]])
@@ -1025,7 +1025,7 @@ class SpiderFootWebUi:
data = dbh.scanResultEventUnique(id, eventType, filterfp)
retdata = []
for row in data:
- escaped = cgi.escape(row[0])
+ escaped = html.escape(row[0])
retdata.append([escaped, row[1], row[2]])
return json.dumps(retdata, ensure_ascii=False)