SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.
Python attacksurfacecticybersecurityfootprintinghackinginformation-gatheringinformation-securityinfosecintelligence-gatheringosintosint-frameworkosint-reconnaissanceosint-toolpentestingpythonreconsecurity-toolsthreat-intelligencethreatintel
Caching for sfp_accounts, content fixes
smicallef7fd057d61acc
diff --git a/dyn/HEADER.tmpl b/dyn/HEADER.tmplindex 9833ec61..d2bae9b4 100644--- a/dyn/HEADER.tmpl+++ b/dyn/HEADER.tmpl@@ -76,7 +76,7 @@<h3>SpiderFoot 3.0</h3></div><div class="modal-body">- <p>SpiderFoot is an open source, GPL-licensed footprinting tool, created by <a href='http://www.binarypool.com' target=_new>Steve Micallef</a>. It is designed to be easy to use, fast and extensible.</p>+ <p>SpiderFoot is an open source, GPL-licensed footprinting tool, created by <a href='https://twitter.com/binarypool' target=_new>Steve Micallef</a>. It is designed to be easy to use, fast and extensible.</p><p>If you have any bugs to report, or requests for enhancements, please contact the support mailing list at <a href=mailto:[email protected]>[email protected]</a>.<br><p>You are running version <b>3.0</b>.</p>diff --git a/modules/sfp__stor_stdout.py b/modules/sfp__stor_stdout.pyindex d6cb13dc..b3443a95 100644--- a/modules/sfp__stor_stdout.py+++ b/modules/sfp__stor_stdout.py@@ -65,8 +65,13 @@ class sfp__stor_stdout(SpiderFootPlugin):srcdata = str(event.sourceEvent.data)if self.opts['_stripnewline']:- data = data.replace("\n", "").replace("\r", "")- srcdata = srcdata.replace("\n", "").replace("\r", "")+ data = data.replace("\n", " ").replace("\r", "")+ srcdata = srcdata.replace("\n", " ").replace("\r", "")++ if "<SFURL>" in data:+ data = data.replace("<SFURL>", "").replace("</SFURL>", "")+ if "<SFURL>" in srcdata:+ srcdata = srcdata.replace("<SFURL>", "").replace("</SFURL>", "")if self.opts['_maxlength'] > 0:data = data[0:self.opts['_maxlength']]diff --git a/modules/sfp_accounts.py b/modules/sfp_accounts.pyindex 5718db61..1aa376d0 100644--- a/modules/sfp_accounts.py+++ b/modules/sfp_accounts.py@@ -29,7 +29,7 @@ class sfp_accounts(SpiderFootPlugin):"ignoreworddict": True,"musthavename": True,"userfromemail": True,- "_maxthreads": 25+ "_maxthreads": 50}# Option descriptions@@ -202,6 +202,15 @@ class sfp_accounts(SpiderFootPlugin):siteList.append(site)i += 1+ if i > 0 and i < self.opts['_maxthreads']:+ data = self.threadSites(name, siteList)+ if data == None:+ return res++ for ret in list(data.keys()):+ if data[ret]:+ res.append(ret)+return res# Handle events sent to this module@@ -228,16 +237,28 @@ class sfp_accounts(SpiderFootPlugin):# If being called for the first time, let's see how trusted the# sites are by attempting to fetch a garbage user.if not self.distrustedChecked:- randpool = 'abcdefghijklmnopqrstuvwxyz1234567890'- randuser = ''.join([random.SystemRandom().choice(randpool) for x in range(10)])- res = self.batchSites(randuser)- if len(res) > 0:+ # Check if a state cache exists first, to not have to do this all the time+ content = self.sf.cacheGet("sfaccounts_state", 72)+ if content:delsites = list()- for site in res:- sitename = site.split(" (Category:")[0]- self.sf.debug("Distrusting " + sitename)- delsites.append(sitename)+ for line in content.split("\n"):+ if line == '':+ continue+ delsites.append(line)self.sites = [d for d in self.sites if d['name'] not in delsites]+ else:+ randpool = 'abcdefghijklmnopqrstuvwxyz1234567890'+ randuser = ''.join([random.SystemRandom().choice(randpool) for x in range(10)])+ res = self.batchSites(randuser)+ if len(res) > 0:+ delsites = list()+ for site in res:+ sitename = site.split(" (Category:")[0]+ self.sf.debug("Distrusting " + sitename)+ delsites.append(sitename)+ self.sites = [d for d in self.sites if d['name'] not in delsites]+ self.sf.cachePut("sfaccounts_state", delsites)+self.distrustedChecked = Trueif eventName == "HUMAN_NAME":diff --git a/sf.py b/sf.pyindex 46cf88a1..159d6e4b 100755--- a/sf.py+++ b/sf.py@@ -114,8 +114,6 @@ if __name__ == '__main__':p.add_argument("-q", action='store_true', help="Disable logging.")args = p.parse_args()- sfConfig['__logstdout'] = True-if args.debug:sfConfig['_debug'] = Trueelse:@@ -128,6 +126,8 @@ if __name__ == '__main__':(addr, port) = args.l.split(":")sfConfig['__webaddr'] = addrsfConfig['__webport'] = int(port)+ else:+ sfConfig['__logstdout'] = TruesfModules = dict()sft = SpiderFoot(sfConfig)@@ -208,7 +208,14 @@ if __name__ == '__main__':sys.exit(-1)target = args.s- targetType = sf.targetType(args.s)+ # Usernames and names - quoted on the commandline - won't have quotes,+ # so add them.+ if " " in target:+ target = "\"" + target + "\""+ if "." not in target and not target.startswith("+") and "\"" not in target:+ target = "\"" + target + "\""+ targetType = sf.targetType(target)+ target = target.strip('"')modlist = list()if not args.t and not args.m:diff --git a/sfcli.py b/sfcli.pyindex 9bd5b468..202b1ad5 100755--- a/sfcli.py+++ b/sfcli.py@@ -1261,7 +1261,7 @@ if __name__ == "__main__":Open Source Intelligence Automation.", plain=True,color=bcolors.GREYBLUE)s.dprint("\- by Steve Micallef | @binarypool\n", plain=True,+ by Steve Micallef | @spiderfoot\n", plain=True,color=bcolors.GREYBLUE_DARK)s.dprint("Version " + s.version + ".")if args.b:diff --git a/sflib.py b/sflib.pyindex aee7a16a..e42f40ec 100644--- a/sflib.py+++ b/sflib.py@@ -423,7 +423,11 @@ class SpiderFoot:fp = open(cacheFile, "w")if type(data) is list:for line in data:- fp.write(line.decode('utf-8') + '\n')+ if type(line) is str:+ fp.write(line)+ fp.write("\n")+ else:+ fp.write(line.decode('utf-8') + '\n')elif type(data) is bytes:fp.write(data.decode('utf-8'))else: