Skip to content

SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.

Caching for sfp_accounts, content fixes

smicallef7fd057d61acc
diff --git a/dyn/HEADER.tmpl b/dyn/HEADER.tmpl
index 9833ec61..d2bae9b4 100644
--- a/dyn/HEADER.tmpl
+++ b/dyn/HEADER.tmpl
@@ -76,7 +76,7 @@
<h3>SpiderFoot 3.0</h3>
</div>
<div class="modal-body">
- <p>SpiderFoot is an open source, GPL-licensed footprinting tool, created by <a href='http://www.binarypool.com' target=_new>Steve Micallef</a>. It is designed to be easy to use, fast and extensible.</p>
+ <p>SpiderFoot is an open source, GPL-licensed footprinting tool, created by <a href='https://twitter.com/binarypool' target=_new>Steve Micallef</a>. It is designed to be easy to use, fast and extensible.</p>
<p>If you have any bugs to report, or requests for enhancements, please contact the support mailing list at <a href=mailto:[email protected]>[email protected]</a>.
<br>
<p>You are running version <b>3.0</b>.</p>
diff --git a/modules/sfp__stor_stdout.py b/modules/sfp__stor_stdout.py
index d6cb13dc..b3443a95 100644
--- a/modules/sfp__stor_stdout.py
+++ b/modules/sfp__stor_stdout.py
@@ -65,8 +65,13 @@ class sfp__stor_stdout(SpiderFootPlugin):
srcdata = str(event.sourceEvent.data)
if self.opts['_stripnewline']:
- data = data.replace("\n", "").replace("\r", "")
- srcdata = srcdata.replace("\n", "").replace("\r", "")
+ data = data.replace("\n", " ").replace("\r", "")
+ srcdata = srcdata.replace("\n", " ").replace("\r", "")
+
+ if "<SFURL>" in data:
+ data = data.replace("<SFURL>", "").replace("</SFURL>", "")
+ if "<SFURL>" in srcdata:
+ srcdata = srcdata.replace("<SFURL>", "").replace("</SFURL>", "")
if self.opts['_maxlength'] > 0:
data = data[0:self.opts['_maxlength']]
diff --git a/modules/sfp_accounts.py b/modules/sfp_accounts.py
index 5718db61..1aa376d0 100644
--- a/modules/sfp_accounts.py
+++ b/modules/sfp_accounts.py
@@ -29,7 +29,7 @@ class sfp_accounts(SpiderFootPlugin):
"ignoreworddict": True,
"musthavename": True,
"userfromemail": True,
- "_maxthreads": 25
+ "_maxthreads": 50
}
# Option descriptions
@@ -202,6 +202,15 @@ class sfp_accounts(SpiderFootPlugin):
siteList.append(site)
i += 1
+ if i > 0 and i < self.opts['_maxthreads']:
+ data = self.threadSites(name, siteList)
+ if data == None:
+ return res
+
+ for ret in list(data.keys()):
+ if data[ret]:
+ res.append(ret)
+
return res
# Handle events sent to this module
@@ -228,16 +237,28 @@ class sfp_accounts(SpiderFootPlugin):
# If being called for the first time, let's see how trusted the
# sites are by attempting to fetch a garbage user.
if not self.distrustedChecked:
- randpool = 'abcdefghijklmnopqrstuvwxyz1234567890'
- randuser = ''.join([random.SystemRandom().choice(randpool) for x in range(10)])
- res = self.batchSites(randuser)
- if len(res) > 0:
+ # Check if a state cache exists first, to not have to do this all the time
+ content = self.sf.cacheGet("sfaccounts_state", 72)
+ if content:
delsites = list()
- for site in res:
- sitename = site.split(" (Category:")[0]
- self.sf.debug("Distrusting " + sitename)
- delsites.append(sitename)
+ for line in content.split("\n"):
+ if line == '':
+ continue
+ delsites.append(line)
self.sites = [d for d in self.sites if d['name'] not in delsites]
+ else:
+ randpool = 'abcdefghijklmnopqrstuvwxyz1234567890'
+ randuser = ''.join([random.SystemRandom().choice(randpool) for x in range(10)])
+ res = self.batchSites(randuser)
+ if len(res) > 0:
+ delsites = list()
+ for site in res:
+ sitename = site.split(" (Category:")[0]
+ self.sf.debug("Distrusting " + sitename)
+ delsites.append(sitename)
+ self.sites = [d for d in self.sites if d['name'] not in delsites]
+ self.sf.cachePut("sfaccounts_state", delsites)
+
self.distrustedChecked = True
if eventName == "HUMAN_NAME":
diff --git a/sf.py b/sf.py
index 46cf88a1..159d6e4b 100755
--- a/sf.py
+++ b/sf.py
@@ -114,8 +114,6 @@ if __name__ == '__main__':
p.add_argument("-q", action='store_true', help="Disable logging.")
args = p.parse_args()
- sfConfig['__logstdout'] = True
-
if args.debug:
sfConfig['_debug'] = True
else:
@@ -128,6 +126,8 @@ if __name__ == '__main__':
(addr, port) = args.l.split(":")
sfConfig['__webaddr'] = addr
sfConfig['__webport'] = int(port)
+ else:
+ sfConfig['__logstdout'] = True
sfModules = dict()
sft = SpiderFoot(sfConfig)
@@ -208,7 +208,14 @@ if __name__ == '__main__':
sys.exit(-1)
target = args.s
- targetType = sf.targetType(args.s)
+ # Usernames and names - quoted on the commandline - won't have quotes,
+ # so add them.
+ if " " in target:
+ target = "\"" + target + "\""
+ if "." not in target and not target.startswith("+") and "\"" not in target:
+ target = "\"" + target + "\""
+ targetType = sf.targetType(target)
+ target = target.strip('"')
modlist = list()
if not args.t and not args.m:
diff --git a/sfcli.py b/sfcli.py
index 9bd5b468..202b1ad5 100755
--- a/sfcli.py
+++ b/sfcli.py
@@ -1261,7 +1261,7 @@ if __name__ == "__main__":
Open Source Intelligence Automation.", plain=True,
color=bcolors.GREYBLUE)
s.dprint("\
- by Steve Micallef | @binarypool\n", plain=True,
+ by Steve Micallef | @spiderfoot\n", plain=True,
color=bcolors.GREYBLUE_DARK)
s.dprint("Version " + s.version + ".")
if args.b:
diff --git a/sflib.py b/sflib.py
index aee7a16a..e42f40ec 100644
--- a/sflib.py
+++ b/sflib.py
@@ -423,7 +423,11 @@ class SpiderFoot:
fp = open(cacheFile, "w")
if type(data) is list:
for line in data:
- fp.write(line.decode('utf-8') + '\n')
+ if type(line) is str:
+ fp.write(line)
+ fp.write("\n")
+ else:
+ fp.write(line.decode('utf-8') + '\n')
elif type(data) is bytes:
fp.write(data.decode('utf-8'))
else: