SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.
Python attacksurfacecticybersecurityfootprintinghackinginformation-gatheringinformation-securityinfosecintelligence-gatheringosintosint-frameworkosint-reconnaissanceosint-toolpentestingpythonreconsecurity-toolsthreat-intelligencethreatintel
UI tidy-ups, bug fixing Junk Files module.
smicallefba5b4fb4a8a2
diff --git a/dyn/newscan.tmpl b/dyn/newscan.tmplindex 6619406c..ddbe1c80 100644--- a/dyn/newscan.tmpl+++ b/dyn/newscan.tmpl@@ -1,18 +1,15 @@<%include file="HEADER.tmpl"/><script src="${docroot}/static/js/spiderfoot.newscan.js"></script><h2>New Scan</h2>+<br><form class="form" action='${docroot}/startscan' method='POST'>- <div class="control-group">- <label class="control-label" for="scanname">Scan Name</label>- <div class="controls">- <input type="text" id="scanname" name="scanname" value="${scanname}" placeholder="Descriptive name for this scan.">- </div>+ <label for="scanname">Scan Name</label>+ <div class="input-group" style='padding-bottom: 10px'>+ <input class='form-control' type="text" id="scanname" name="scanname" value="${scanname}" placeholder="Descriptive name for this scan."></div>- <div class="control-group">- <label class="control-label" for="scantarget">Seed Target</label>- <div class="controls">- <input type="text" value="${scantarget}" data-toggle='popover' data-html=true data-animation=true data-title='Usage' data-content='The <i>Seed Target</i> can be one of the following. SpiderFoot will automatically detect the target type based on the format of your input.<br><br><b>Domain Name</b>: e.g. <i>example.com</i><br><b>IPv4 Address</b>: e.g. <i>1.2.3.4</i><br><b>IPv6 Address</b>: e.g. <i>2606:4700:4700::1111</i><br><b>Hostname/Sub-domain</b>: e.g. <i>abc.example.com</i><br><b>Subnet</b>: e.g. <i>1.2.3.0/24</i><br><b>ASN</b>: e.g. <i>1234</i><br><b>E-mail address</b>: e.g. <i>[email protected]</i><br><b>Phone Number</b>: e.g. <i>+12345678901</i> (E.164 format)<br><b>Human Name</b>: e.g. <i>"John Smith"</i> (must be in quotes)<br><b>Username</b>: e.g. <i>"jsmith2000"</i> (must be in quotes)' data-placement='right' id="scantarget" name="scantarget" placeholder="Starting point for the scan.">- </div>+ <label for="scantarget">Seed Target</label>+ <div class="input-group">+ <input class='form-control' type="text" value="${scantarget}" data-toggle='popover' data-html=true data-animation=true data-title='Usage' data-content='The <i>Seed Target</i> can be one of the following. SpiderFoot will automatically detect the target type based on the format of your input.<br><br><b>Domain Name</b>: e.g. <i>example.com</i><br><b>IPv4 Address</b>: e.g. <i>1.2.3.4</i><br><b>IPv6 Address</b>: e.g. <i>2606:4700:4700::1111</i><br><b>Hostname/Sub-domain</b>: e.g. <i>abc.example.com</i><br><b>Subnet</b>: e.g. <i>1.2.3.0/24</i><br><b>ASN</b>: e.g. <i>1234</i><br><b>E-mail address</b>: e.g. <i>[email protected]</i><br><b>Phone Number</b>: e.g. <i>+12345678901</i> (E.164 format)<br><b>Human Name</b>: e.g. <i>"John Smith"</i> (must be in quotes)<br><b>Username</b>: e.g. <i>"jsmith2000"</i> (must be in quotes)' data-placement='right' id="scantarget" name="scantarget" placeholder="Starting point for the scan."></div><p></p>@@ -22,9 +19,9 @@<li id='usetab' class="active"><a href='#'>By Use Case</a></li><li id='typetab'><a href='#'>By Required Data</a></li><li id='moduletab'><a href='#'>By Module</a></li>- <div class='text-right' id='selectors' style='display: none'>- <button id="btn-select-all" type="button" class="btn-mini btn-info">Select All</button> - <button id="btn-deselect-all" type="button" class="btn-mini btn-info">De-Select All</button>+ <div class='btn-group-sm pull-right' role="group" id='selectors' style='display: none'>+ <button id="btn-select-all" type="button" class="btn btn-info">Select All</button> + <button id="btn-deselect-all" type="button" class="btn btn-info">De-Select All</button></div></ul><table class="table table-striped table-condensed" id="usetable">diff --git a/dyn/opts.tmpl b/dyn/opts.tmplindex ce86b638..a8281ee0 100644--- a/dyn/opts.tmpl+++ b/dyn/opts.tmpl@@ -46,12 +46,16 @@<thead><tr> <th>Option</th> <th>Value</th></tr></thead><tbody>% for opt in sorted(opts.keys()):% if not opt.startswith("__"):- <tr><td>${opts['__globaloptdescs__'][opt]}</td><td>+ <tr><td>${opts['__globaloptdescs__'][opt]}</td><td style='width: 75%;vertical-align: middle'>% if type(opts[opt]) is int:- <input id='${opt}' class='input-mini' type=text value="${opts[opt]}">+ <div class='input-group-lg' style='width: 100px'>+ <input id='${opt}' class='form-control' type=text value="${opts[opt]}">+ </div>% endif% if type(opts[opt]) is str:- <input id='${opt}' class='input-xxlarge' type=text value="${opts[opt]}">+ <div class='input-group-lg'>+ <input id='${opt}' class='form-control' type=text value="${opts[opt]}">+ </div>% endif% if type(opts[opt]) is bool:<%@@ -69,11 +73,15 @@% endif% if type(opts[opt]) is list and type(opts[opt][0]) is str:<% expandedopt = ','.join(opts[opt]) %>- <input id='${opt}' class='input-xxlarge' type=text value="${expandedopt}">+ <div class='input-group-lg'>+ <input id='${opt}' class='form-control' type=text value="${expandedopt}">+ </div>% endif% if type(opts[opt]) is list and type(opts[opt][0]) is int:<% expandedopt = ','.join(str(x) for x in opts[opt]) %>- <input id='${opt}' class='input-xxlarge' type=text value="${expandedopt}">+ <div class='input-group-lg' style='width: 100px'>+ <input id='${opt}' class='form-control' type=text value="${expandedopt}">+ </div>% endif</td></tr>% endif@@ -98,12 +106,16 @@optdesc = opts['__modules__'][mod]['optdescs'][opt]%>- <tr><td>${optdesc}</td><td>+ <tr><td>${optdesc}</td><td style='width: 75%;vertical-align: middle'>% if type(modopts[opt]) is int:- <input id='${mod}:${opt}' class='input-mini' type=text value="${modopts[opt]}">+ <div class='input-group-lg' style='width: 100px'>+ <input id='${mod}:${opt}' class='form-control' type=text value="${modopts[opt]}">+ </div>% endif% if type(modopts[opt]) is str:- <input id='${mod}:${opt}' class='input-xxlarge' type=text value="${modopts[opt]}">+ <div class='input-group-lg'>+ <input id='${mod}:${opt}' class='form-control' type=text value="${modopts[opt]}">+ </div>% endif% if type(modopts[opt]) is bool:<%@@ -121,11 +133,15 @@% endif% if type(modopts[opt]) is list and type(modopts[opt][0]) is str:<% expandedopt = ','.join(modopts[opt]) %>- <input id='${mod}:${opt}' class='input-xxlarge' type=text value="${expandedopt}">+ <div class='input-group-lg'>+ <input id='${mod}:${opt}' class='form-control' type=text value="${expandedopt}">+ </div>% endif% if type(modopts[opt]) is list and type(modopts[opt][0]) is int:<% expandedopt = ','.join(str(x) for x in modopts[opt]) %>- <input id='${mod}:${opt}' class='input-xxlarge' type=text value="${expandedopt}">+ <div class='input-group-lg' style='width: 100px'>+ <input id='${mod}:${opt}' class='form-control' type=text value="${expandedopt}">+ </div>% endif</td></tr>diff --git a/dyn/scaninfo.tmpl b/dyn/scaninfo.tmplindex a6e8391d..693e8676 100644--- a/dyn/scaninfo.tmpl+++ b/dyn/scaninfo.tmpl@@ -9,8 +9,8 @@<a id='btn-info' class="btn btn-default" onClick='viewScanConfig("${id}")'><i class='glyphicon glyphicon-cog'></i> Scan Settings</a><a id='btn-log' class="btn btn-default" onClick='viewScanLog("${id}")'><i class='glyphicon glyphicon-book'></i> Log</a></div>- <div class='pull-right'>- <div id='btn-search' class='input-group'>+ <div class='btn-group pull-right'>+ <div id='btn-search' class='input-group pull-right'><input class='form-control' id='searchvalue' type='text' placeholder='Search...' data-toggle='popover' data-html='true' data-animation='true' data-title='Usage' data-content="Supply a regular expression encapsulated in '/', e.g. <i>/searchregex/</i> or a simple wildcard search, e.g. <i>*string*</i> or an exact value, e.g. <i>string</i>.<br><br><b>Note</b> that the current scope is what's searched."><div class='input-group-btn'><button class='btn btn-primary' id='searchbutton' type='button' onClick="$('#btn-search').popover('toggle');searchDirector('${id}')">@@ -19,14 +19,11 @@</div></div></div>- <div class='pull-right'>- <div class="btn-group" role="group">- <a id='btn-refresh' rel="tooltip" title="Refresh" class="btn-success btn btn-default" onClick='refresh()'><i class='glyphicon glyphicon-refresh glyphicon glyphicon-white'></i></a>- <a id='btn-export' rel="tooltip" title="Export Data" class="btn-success btn btn-default" onClick='exportEventData("${id}", currentType)'><i class='glyphicon glyphicon-download-alt glyphicon glyphicon-white'></i></a>+ <div class="btn-group pull-right" role="group">+ <button id='btn-refresh' rel="tooltip" title="Refresh" class="btn btn-success btn-default" onClick='refresh()'><i class='glyphicon glyphicon-refresh glyphicon glyphicon-white'></i></button>+ <button id='btn-export' rel="tooltip" title="Export Data" class="btn btn-success btn-default" onClick='exportEventData("${id}", currentType)'><i class='glyphicon glyphicon-download-alt glyphicon glyphicon-white'></i></button></div>- </div>- <div class='pull-right'>- <div id='customtabview' role="group" class="btn-group">+ <div id='customtabview' role="group" class="btn-group pull-right"><a id='btn-fullview' rel="tooltip" data-title="Full Data View" class="btn btn-default active" onClick='browseUpdate("full")'><i class='glyphicon glyphicon-th glyphicon glyphicon-white'></i></a><a id='btn-uniqueview' rel="tooltip" data-title="Unique Data View" class="btn btn-default" onClick='browseUpdate("unique")'><i class='glyphicon glyphicon-align-justify glyphicon glyphicon-white'></i></a><a id='btn-vizview' rel="tooltip" data-title="Visualize" class="btn btn-default dropdown-toggle" data-toggle="dropdown"><i class='glyphicon glyphicon-picture glyphicon glyphicon-white'></i></a>@@ -36,23 +33,18 @@<li><a class='link' onClick='browseUpdate("viz-dendro")'>Discovery Path</a></li></ul></div>- </div>- <div class='pull-right'>- <div id='customvizview' class="btn-group">+ <div id='customvizview' class="btn-group pull-right"><a id='btn-random' rel="tooltip" data-title="Random Layout" class="btn btn-default active" onClick='vizUpdate("random")'><b>R</b></a><a id='btn-forceatlas' rel="tooltip" data-title="Force Layout" class="btn btn-default" onClick='vizUpdate("force")'><b>F</b></a><a id='btn-saveimage' rel="tooltip" data-title="Save Image" class="btn btn-default dropdown-toggle" onClick='vizUpdate("download")'><i class='glyphicon glyphicon-picture glyphicon glyphicon-white'></i></a></div>- </div>- <div class='pull-right'>- <div id='modifyactions' role="group" class="btn-group">+ <div id='modifyactions' role="group" class="btn-group pull-right"><a id='btn-setfp' rel="tooltip" data-title="Set/Unset False Positive flag" class="btn-warning btn btn-default dropdown-toggle" data-toggle="dropdown"><i class='glyphicon glyphicon-ban-circle glyphicon glyphicon-white'></i></a><ul class="dropdown-menu"><li><a href='#' class='link' onClick='setFp(1)'>Set False Positive flag</a></li><li><a href='#' class='link' onClick='setFp(0)'>Unset False Positive flag</a></li></ul></div>- </div></div><script src="${docroot}/static/js/sigma.min.js"></script><script src="${docroot}/static/js/sigma.parsers.json.min.js"></script>@@ -162,11 +154,11 @@lastSearchType = typeName;lastSearchQuery = query;var crumbs = " <ul class='breadcrumb' id='breadcrumbs'> <li><a class='link' onClick='browseEventList(\"" + instanceId + "\");'>Browse</a>";- crumbs += " <span class='divider'>></span></li>";+ crumbs += " <span class='divider'></span></li>";if (typeName != null) {crumbs += " <li><a class='link' onClick=";crumbs += "'browseEventData(\"" + instanceId + "\",\"" + currentTypeName + "\",\"" + typeName + "\",\"full\");'>";- crumbs += unescape(currentTypeName) + "</a><span class='divider'>></span></li>";+ crumbs += unescape(currentTypeName) + "</a><span class='divider'></span></li>";}crumbs += "<li>Search results";crumbs += " (" + data.length + " records)</li></ul>";@@ -348,7 +340,7 @@dataloaders.push(function() {sf.fetchData('${docroot}/scanlog', {'id': instanceId, 'limit': 10}, function(data) {- var table = "<div class='well'><h4>Recent Logs</h4><table id='statuslog-content' class='table table-bordered table-striped table-condensed font-size-90-percent'>";+ var table = "<div><h4>Recent Logs</h4><table id='statuslog-content' class='table table-bordered table-striped table-condensed font-size-90-percent'>";table += "<thead><tr><th>Time</th><th>Component</th><th>Type</th><th>Event</th></tr></thead><tbody>";for (var i = 0; i < data.length; i++) {if (data[i][2] == "ERROR") {@@ -478,12 +470,12 @@}}var crumbs = " <ul class='breadcrumb' id='breadcrumbs'> <li><a class='link' onClick='browseEventList(\"" + instanceId + "\");'>Browse</a>";- crumbs += " <span class='divider'>></span></li> <li><a class='link' onClick=";+ crumbs += " <span class='divider'></span></li> <li><a class='link' onClick=";crumbs += "'browseEventData(\"" + instanceId + "\",\"" + eventTypeLabel + "\",\"" + eventType + "\",\"";crumbs += format + "\", " + filterFP + ");'>";crumbs += unescape(eventTypeLabel) + "</a></li>";if (fpcount > 0) {- crumbs += "<li class='pull-right text-center'><i class='glyphicon glyphicon-ban-circle' /> Hide " + fpcount + " False Positives: ";+ crumbs += "<div class='pull-right text-center'><i class='glyphicon glyphicon-ban-circle'></i> Hide " + fpcount + " False Positives: ";crumbs += "<input class='vertical-align-top' type='checkbox' ";crumbs += "onClick=\"browseEventData('" + instanceId + "', '" + eventTypeLabel +"', '" + eventType + "'";crumbs += ",'" + format + "', ";@@ -494,7 +486,7 @@fp = 0;ch = "checked";}- crumbs += fp + ")\" " + ch + "></input></li>";+ crumbs += fp + ")\" " + ch + "></input></div>";}crumbs += "</ul>";@@ -566,11 +558,11 @@}sf.fetchData('${docroot}/scaneventresultsunique', {'id': instanceId, 'eventType': eventType, 'filterfp': filterFP }, function(data) {var crumbs = " <ul class='breadcrumb' id='breadcrumbs'> <li><a class='link' onClick='browseEventList(\"" + instanceId + "\");'>Browse</a>";- crumbs += " <span class='divider'>></span></li> <li><a class='link' onClick=";+ crumbs += " <span class='divider'></span></li> <li><a class='link' onClick=";crumbs += "'browseEventData(\"" + instanceId + "\",\"" + eventTypeLabel + "\",\"" + eventType + "\",\"" + format;crumbs += "\", " + filterFP + ");'>";crumbs += unescape(eventTypeLabel) + "</a></li>";- crumbs += "<li class='pull-right text-center'><i class='glyphicon glyphicon-ban-circle' /> Hide False Positives: ";+ crumbs += "<div class='pull-right text-center'><i class='glyphicon glyphicon-ban-circle'></i> Hide False Positives: ";crumbs += "<input class='vertical-align-top' type='checkbox' ";crumbs += "onClick=\"browseEventData('" + instanceId + "', '" + eventTypeLabel +"', '" + eventType + "'";crumbs += ",'" + format + "', ";@@ -581,7 +573,7 @@fp = 0;ch = "checked";}- crumbs += fp + ")\" " + ch + "></input></li>";+ crumbs += fp + ")\" " + ch + "></input></div>";crumbs += "</ul>";var table = "<table id='scansummary-content' class='table table-bordered table-striped small'>";@@ -606,7 +598,7 @@if (format.indexOf("viz-bubble") == 0) {sf.fetchData('${docroot}/scaneventresults', {'id': instanceId, 'eventType': eventType }, function(data) {var crumbs = " <ul class='breadcrumb' id='breadcrumbs'> <li><a class='link' onClick='browseEventList(\"" + instanceId + "\");'>Browse</a>";- crumbs += " <span class='divider'>></span></li> <li><a class='link' onClick=";+ crumbs += " <span class='divider'>;</span></li> <li><a class='link' onClick=";crumbs += "'browseEventData(\"" + instanceId + "\",\"" + eventTypeLabel + "\",\"" + eventType + "\",\"" + format + "\");'>";crumbs += unescape(eventTypeLabel) + "</a></li></ul>";itemList = []@@ -627,7 +619,7 @@if (format == "viz-dendro") {sf.fetchData("${docroot}/scanelementtypediscovery", {'id': instanceId, 'eventType': eventType }, function(data) {var crumbs = " <ul class='breadcrumb' id='breadcrumbs'> <li><a class='link' onClick='browseEventList(\"" + instanceId + "\");'>Browse</a>";- crumbs += " <span class='divider'>></span></li> <li><a class='link' onClick=";+ crumbs += " <span class='divider'></span></li> <li><a class='link' onClick=";crumbs += "'browseEventData(\"" + instanceId + "\",\"" + eventTypeLabel + "\",\"" + eventType + "\",\"" + format + "\");'>";crumbs += unescape(eventTypeLabel) + "</a></li></ul>";diff --git a/dyn/scanlist.tmpl b/dyn/scanlist.tmplindex 3a74fb17..e9d4e1ad 100644--- a/dyn/scanlist.tmpl+++ b/dyn/scanlist.tmpl@@ -1,5 +1,5 @@<%include file="HEADER.tmpl"/>- <h2>Scans <img id=loader src=${docroot}/static/img/loader.gif></h2>+ <h2>Scans <img id="loader" src="${docroot}/static/img/loader.gif"></h2><div id="scancontent">% if newscan:<div class="alert alert-success">diff --git a/modules/sfp_junkfiles.py b/modules/sfp_junkfiles.pyindex f0175bf0..fe4e0b69 100644--- a/modules/sfp_junkfiles.py+++ b/modules/sfp_junkfiles.py@@ -134,13 +134,8 @@ class sfp_junkfiles(SpiderFootPlugin):else:self.bases[base] = True- # don't do anything with the root directory of a site- self.sf.debug("Base: " + base + ", event: " + eventData)- if base == eventData + "/" or base == eventData:- return None-- # http://www/blah/abc.html -> try http://www/blah.[dirs]- for dirfile in self.opts['dirs']:+ # http://www/blah/abc.html -> try http://www/blah/[files]+ for f in self.opts['files']:if self.checkForStop():return None@@ -148,12 +143,8 @@ class sfp_junkfiles(SpiderFootPlugin):self.sf.debug("Skipping " + host + " because it doesn't return 404s.")return None- if base.count('/') == 3:- self.sf.debug("Skipping base url.")- continue-- self.sf.debug("Trying " + dirfile + " against " + eventData)- fetch = base[0:len(base) - 1] + "." + dirfile+ self.sf.debug("Trying " + f + " against " + eventData)+ fetch = base + fif fetch not in self.results:self.results[fetch] = Trueelse:@@ -173,8 +164,13 @@ class sfp_junkfiles(SpiderFootPlugin):evt = SpiderFootEvent("JUNK_FILE", fetch, self.__name__, event)self.notifyListeners(evt)- # http://www/blah/abc.html -> try http://www/blah/[files]- for f in self.opts['files']:+ # don't do anything with the root directory of a site+ self.sf.debug("Base: " + base + ", event: " + eventData)+ if base == eventData + "/" or base == eventData:+ return None++ # http://www/blah/abc.html -> try http://www/blah.[dirs]+ for dirfile in self.opts['dirs']:if self.checkForStop():return None@@ -182,8 +178,12 @@ class sfp_junkfiles(SpiderFootPlugin):self.sf.debug("Skipping " + host + " because it doesn't return 404s.")return None- self.sf.debug("Trying " + f + " against " + eventData)- fetch = base + f+ if base.count('/') == 3:+ self.sf.debug("Skipping base url.")+ continue++ self.sf.debug("Trying " + dirfile + " against " + eventData)+ fetch = base[0:len(base) - 1] + "." + dirfileif fetch not in self.results:self.results[fetch] = Trueelse:@@ -203,6 +203,4 @@ class sfp_junkfiles(SpiderFootPlugin):evt = SpiderFootEvent("JUNK_FILE", fetch, self.__name__, event)self.notifyListeners(evt)- return None-# End of sfp_junkfiles classdiff --git a/sfwebui.py b/sfwebui.pyindex 9d263099..cbab2c25 100644--- a/sfwebui.py+++ b/sfwebui.py@@ -553,10 +553,11 @@ class SpiderFootWebUi:scaninfo.exposed = True# Settings- def opts(self):+ def opts(self, updated=None):templ = Template(filename='dyn/opts.tmpl', lookup=self.lookup)self.token = random.SystemRandom().randint(0, 99999999)- return templ.render(opts=self.config, pageid='SETTINGS', token=self.token, docroot=self.docroot)+ return templ.render(opts=self.config, pageid='SETTINGS', token=self.token,+ updated=updated, docroot=self.docroot)opts.exposed = True@@ -697,8 +698,7 @@ class SpiderFootWebUi:except Exception as e:return self.error("Processing one or more of your inputs failed: " + str(e))-- raise cherrypy.HTTPRedirect("/opts")+ raise cherrypy.HTTPRedirect("/opts?updated=1")savesettings.exposed = Truediff --git a/static/js/spiderfoot.scanlist.js b/static/js/spiderfoot.scanlist.jsindex c51c22dd..0d3884d9 100644--- a/static/js/spiderfoot.scanlist.js+++ b/static/js/spiderfoot.scanlist.js@@ -213,15 +213,9 @@table += '</th></tr></tfoot>';table += "</table>";- footer = "<div>Total Scans: <b>" + i + "</b>";- if (filtered > 0) {- footer += " (" + filtered + " filtered out)";- }- footer += "</div>";-$("#loader").fadeOut(500);$("#scancontent-wrapper").remove();- $("#scancontent").append("<div id='scancontent-wrapper'> " + buttons + table + footer + "</div>");+ $("#scancontent").append("<div id='scancontent-wrapper'> " + buttons + table + "</div>");sf.updateTooltips();$("#scanlist").tablesorter().tablesorterPager({container: $(".ts-pager"),diff --git a/static/js/viz.js b/static/js/viz.jsindex 54c05fcd..b420062a 100644--- a/static/js/viz.js+++ b/static/js/viz.js@@ -112,7 +112,7 @@ function sf_viz_vbar(targetId, gdata) {data[i] = sf_viz_vbar_type(gdata[i])}x.domain(data.map(function(d) { return d.name; }));- y.domain([0, d3.max(data, function(d) { return d.pct; })]);+ y.domain([0, d3.max(data, function(d) { return d.pct*100; })]);svg.append("g").attr("class", "x axis")@@ -142,8 +142,8 @@ function sf_viz_vbar(targetId, gdata) {.attr("class", "bar").attr("x", function(d) { return x(d.name); }).attr("width", x.rangeBand())- .attr("y", function(d) { return y(d.pct); })- .attr("height", function(d) { return height - y(d.pct); })+ .attr("y", function(d) { return y(d.pct*100); })+ .attr("height", function(d) { return height - y(d.pct*100); }).on('mousedown', function(d) { showToolTip(" ",0,0,false); d.link(d); } ).on("mouseover", function(d, i) {showToolTip(buildPopupMessage(d), d3.event.pageX+10, d3.event.pageY+10,true);