Skip to content

SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.

Update style for pep8

Brendan Colesccdb1e525f1c
diff --git a/sf.py b/sf.py
index 902a7681..12a46e08 100755
--- a/sf.py
+++ b/sf.py
@@ -37,8 +37,8 @@ from sfscan import SpiderFootScanner
# be overridden from saved configuration settings stored in the DB.
sfConfig = {
'_debug': False, # Debug
- '__logging': True, # Logging in general
- '__outputfilter': None, # Event types to filter from modules' output
+ '__logging': True, # Logging in general
+ '__outputfilter': None, # Event types to filter from modules' output
'__blocknotif': False, # Block notifications
'_fatalerrors': False,
'_useragent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:62.0) Gecko/20100101 Firefox/62.0', # User-Agent to use for HTTP requests
@@ -85,6 +85,7 @@ sfOptdescs = {
scanId = None
dbh = None
+
def handle_abort(signal, frame):
"""handle interrupt and abort scan."""
print("[*] Aborting...")
@@ -92,6 +93,7 @@ def handle_abort(signal, frame):
dbh.scanInstanceSet(scanId, None, None, "ABORTED")
sys.exit(-1)
+
if __name__ == '__main__':
if len(sys.argv) == 0:
print("SpiderFoot 3.x now requires -l <ip>:<port> to start the web server.")
@@ -358,7 +360,7 @@ if __name__ == '__main__':
if args.D:
delim = args.D
else:
- if args.o in [ "tab", None ]:
+ if args.o in ["tab", None]:
delim = "\t"
if args.o == "csv":
@@ -380,7 +382,7 @@ if __name__ == '__main__':
if not info:
time.sleep(1)
continue
- if info[5] in [ "ERROR-FAILED", "ABORT-REQUESTED", "ABORTED", "FINISHED" ]:
+ if info[5] in ["ERROR-FAILED", "ABORT-REQUESTED", "ABORTED", "FINISHED"]:
if sfConfig['__logging']:
print("[*] Scan completed with status " + info[5])
if args.o == "json":
@@ -451,11 +453,11 @@ if __name__ == '__main__':
print("Warning: passwd file contains no passwords. Authentication disabled.")
print("********************************************************************")
else:
- print("")
- print("********************************************************************")
- print("Please consider adding authentication to protect this instance!")
- print("Refer to https://www.spiderfoot.net/documentation/#security.")
- print("********************************************************************")
+ print("")
+ print("********************************************************************")
+ print("Please consider adding authentication to protect this instance!")
+ print("Refer to https://www.spiderfoot.net/documentation/#security.")
+ print("********************************************************************")
key_path = sf.dataPath() + '/spiderfoot.key'
crt_path = sf.dataPath() + '/spiderfoot.crt'
diff --git a/sfcli.py b/sfcli.py
index ac607b3b..77b16616 100755
--- a/sfcli.py
+++ b/sfcli.py
@@ -28,6 +28,7 @@ try:
except:
import pyreadline as readline
+
# Colors to make things purty
class bcolors:
GREYBLUE = '\x1b[38;5;25m'
@@ -38,6 +39,7 @@ class bcolors:
ENDC = '\033[0m'
GREYBLUE_DARK = '\x1b[38;5;24m'
+
class SpiderFootCli(cmd.Cmd):
version = "3.2-DEV"
pipecmd = None
@@ -125,8 +127,7 @@ class SpiderFootCli(cmd.Cmd):
if err or not self.ownopts["cli.silent"]:
if not plain or color:
- cout = col + bcolors.BOLD + pfx + " " + bcolors.ENDC + \
- col + msg + bcolors.ENDC
+ cout = col + bcolors.BOLD + pfx + " " + bcolors.ENDC + col + msg + bcolors.ENDC
# Never include color in the spool
sout = pfx + " " + msg
else:
@@ -165,7 +166,7 @@ class SpiderFootCli(cmd.Cmd):
def do_history(self, line):
"""history [-l]
- Short-cut command for set cli.history = 1/0.
+ Short-cut command for set cli.history = 1/0.
Add -l to just list the history."""
c = self.myparseline(line)
@@ -242,12 +243,11 @@ class SpiderFootCli(cmd.Cmd):
# we have a dict key
cn = c
v = str(r[c])
- #print(str(cn) + ", " + str(c) + ", " + str(v))
+ # print(str(cn) + ", " + str(c) + ", " + str(v))
if len(v) > maxsize.get(cn, 0):
maxsize[cn] = len(v)
i += 1
-
# Adjust for long titles
if titlemap:
for c in maxsize:
@@ -261,32 +261,32 @@ class SpiderFootCli(cmd.Cmd):
t = titlemap.get(c, c)
else:
t = c
- #out += t
+ # out += t
out.append(t)
sdiff = maxsize[c] - len(t) + 1
- #out += " " * spaces
+ # out += " " * spaces
out.append(" " * spaces)
- if sdiff > 0 and i < len(cols)-1:
- #out += " " * sdiff
+ if sdiff > 0 and i < len(cols) - 1:
+ # out += " " * sdiff
out.append(" " * sdiff)
i += 1
- #out += "\n"
+ # out += "\n"
out.append('\n')
# Then the separator
i = 0
for c in cols:
- #out += "-" * ((maxsize[c]+spaces))
- out.append("-" * ((maxsize[c]+spaces)))
- if i < len(cols)-1:
- #out += "+"
+ # out += "-" * ((maxsize[c]+spaces))
+ out.append("-" * ((maxsize[c] + spaces)))
+ if i < len(cols) - 1:
+ # out += "+"
out.append("+")
i += 1
- #out += "\n"
+ # out += "\n"
out.append("\n")
# Then the actual data
- #ts = time.time()
+ # ts = time.time()
for r in data:
i = 0
di = 0
@@ -317,19 +317,18 @@ class SpiderFootCli(cmd.Cmd):
sdiff = (maxsize[cn] - lv) + spaces
else:
sdiff = (maxsize[cn] - lv) + spaces - 1
- if di < len(cols)-1:
- #out += " " * sdiff
+ if di < len(cols) - 1:
+ # out += " " * sdiff
out.append(" " * sdiff)
- if di < len(cols)-1:
- #out += "| "
+ if di < len(cols) - 1:
+ # out += "| "
out.append("| ")
di += 1
i += 1
- #out += "\n"
+ # out += "\n"
out.append("\n")
- #print("time: " + str(time.time() - ts))
- #return ""
+ # print("time: " + str(time.time() - ts))
return ''.join(out)
# Make a request to the SpiderFoot server
@@ -342,11 +341,11 @@ class SpiderFootCli(cmd.Cmd):
self.edprint("Invalid request URL: %s" % url)
return None
- #logging.basicConfig()
- #logging.getLogger().setLevel(logging.DEBUG)
- #requests_log = logging.getLogger("requests.packages.urllib3")
- #requests_log.setLevel(logging.DEBUG)
- #requests_log.propagate = True
+ # logging.basicConfig()
+ # logging.getLogger().setLevel(logging.DEBUG)
+ # requests_log = logging.getLogger("requests.packages.urllib3")
+ # requests_log.setLevel(logging.DEBUG)
+ # requests_log.propagate = True
try:
headers = {
"User-agent": "SpiderFoot-CLI/" + self.version,
@@ -356,26 +355,28 @@ class SpiderFootCli(cmd.Cmd):
self.ddprint("Fetching: " + url)
self.ddprint("Posting: " + str(post))
if not post:
- r = requests.get(url,
- headers = headers,
- verify = self.ownopts['cli.ssl_verify'],
- auth = requests.auth.HTTPDigestAuth(
- self.ownopts['cli.username'],
- self.ownopts['cli.password']
- )
+ r = requests.get(
+ url,
+ headers=headers,
+ verify=self.ownopts['cli.ssl_verify'],
+ auth=requests.auth.HTTPDigestAuth(
+ self.ownopts['cli.username'],
+ self.ownopts['cli.password']
)
+ )
else:
- r = requests.post(url,
- headers = headers,
- verify = self.ownopts['cli.ssl_verify'],
- auth = requests.auth.HTTPDigestAuth(
- self.ownopts['cli.username'],
- self.ownopts['cli.password']
- ),
- data = post
- )
+ r = requests.post(
+ url,
+ headers=headers,
+ verify=self.ownopts['cli.ssl_verify'],
+ auth=requests.auth.HTTPDigestAuth(
+ self.ownopts['cli.username'],
+ self.ownopts['cli.password']
+ ),
+ data=post
+ )
self.ddprint("Response: " + str(r))
- if r.status_code == requests.codes.ok: #pylint: disable=no-member
+ if r.status_code == requests.codes.ok: # pylint: disable=no-member
return r.text
else:
r.raise_for_status()
@@ -410,7 +411,7 @@ class SpiderFootCli(cmd.Cmd):
ret[1] = list()
i = 0
ret[1].append(list())
- for t in s[(s.index('|')+1):]:
+ for t in s[(s.index('|') + 1):]:
if t == '|':
i += 1
ret[1].append(list())
@@ -467,7 +468,7 @@ class SpiderFootCli(cmd.Cmd):
return
pipecmd = pc[0]
pipeargs = " ".join(pc[1:])
- if pipecmd not in [ "str", "regex", "file", "grep", "top", "last" ]:
+ if pipecmd not in ["str", "regex", "file", "grep", "top", "last"]:
self.edprint("Unrecognised pipe command.")
return
@@ -548,7 +549,7 @@ class SpiderFootCli(cmd.Cmd):
if s[1] != self.version:
self.edprint("Server and CLI version are not the same (" + s[1] + \
- " / " + self.version + "). This could lead to unpredictable results!")
+ " / " + self.version + "). This could lead to unpredictable results!")
# List all SpiderFoot modules.
def do_modules(self, line, cacheonly=False):
@@ -570,15 +571,24 @@ class SpiderFootCli(cmd.Cmd):
"""types
List all available element types and their descriptions."""
d = self.request(self.ownopts['cli.server_baseurl'] + "/eventtypes")
+
if not d:
return
+
if cacheonly:
j = json.loads(d)
for t in j:
self.types.append(t[0])
return
- self.send_output(d, line, titles={ "1": "Element description",
- "0": "Element name"})
+
+ self.send_output(
+ d,
+ line,
+ titles={
+ "1": "Element description",
+ "0": "Element name"
+ }
+ )
# Load commands from a file.
def do_load(self, line):
@@ -589,7 +599,7 @@ class SpiderFootCli(cmd.Cmd):
# Get scan info and config.
def do_scaninfo(self, line):
"""scaninfo <sid> [-c]
- Get status information for scan ID <sid>, optionally also its
+ Get status information for scan ID <sid>, optionally also its
configuration if -c is supplied."""
c = self.myparseline(line)
if len(c[0]) < 1:
@@ -605,7 +615,7 @@ class SpiderFootCli(cmd.Cmd):
self.dprint("No such scan exists.")
return
- out = "Name: " + j['meta'][0] + "\n"
+ out = "Name: " + j['meta'][0] + "\n"
out += "ID: " + sid + "\n"
out += "Target: " + j['meta'][1] + "\n"
out += "Started: " + j['meta'][3] + "\n"
@@ -634,20 +644,21 @@ class SpiderFootCli(cmd.Cmd):
c = self.myparseline(line)
titles = dict()
if "-x" in c[0]:
- titles = {"0": "ID",
- "1": "Name",
- "2": "Target",
- "4": "Started",
- "5": "Finished",
- "6": "Status",
- "7": "Total Elements"
+ titles = {
+ "0": "ID",
+ "1": "Name",
+ "2": "Target",
+ "4": "Started",
+ "5": "Finished",
+ "6": "Status",
+ "7": "Total Elements"
}
else:
titles = {
- "0": "ID",
- "2": "Target",
- "6": "Status",
- "7": "Total Elements"
+ "0": "ID",
+ "2": "Target",
+ "6": "Status",
+ "7": "Total Elements"
}
self.send_output(d, line, titles=titles)
@@ -655,18 +666,18 @@ class SpiderFootCli(cmd.Cmd):
# Show the data from a scan.
def do_data(self, line):
"""data <sid> [-t type] [-x] [-u]
- Get the scan data for scan ID <sid> and optionally the element
- type [type] (e.g. EMAILADDR), [type]. Use -x for extended format.
+ Get the scan data for scan ID <sid> and optionally the element
+ type [type] (e.g. EMAILADDR), [type]. Use -x for extended format.
Use -u for a unique set of results."""
c = self.myparseline(line)
if len(c[0]) < 1:
self.edprint("Invalid syntax.")
return
- post = { "id": c[0][0] }
+ post = {"id": c[0][0]}
if "-t" in c[0]:
- post["eventType"] = c[0][c[0].index("-t")+1]
+ post["eventType"] = c[0][c[0].index("-t") + 1]
else:
post["eventType"] = "ALL"
@@ -684,13 +695,13 @@ class SpiderFootCli(cmd.Cmd):
return
if "-u" in c[0]:
- titles={
- "0": "Data"
+ titles = {
+ "0": "Data"
}
else:
- titles={
- "10": "Type",
- "1": "Data"
+ titles = {
+ "10": "Type",
+ "1": "Data"
}
if "-x" in c[0]:
titles["0"] = "Last Seen"
@@ -711,16 +722,16 @@ class SpiderFootCli(cmd.Cmd):
return
if '-t' in c[0]:
- export_format = c[0][c[0].index("-t")+1]
+ export_format = c[0][c[0].index("-t") + 1]
else:
export_format = 'json'
- if not export_format in ['json', 'csv']:
+ if export_format not in ['json', 'csv']:
print("Invalid export format: %s" % export_format)
return
base_url = self.ownopts['cli.server_baseurl']
- post = { "ids": c[0][0] }
+ post = {"ids": c[0][0]}
if export_format == 'json':
res = self.request(base_url + '/scanexportjsonmulti', post=post)
@@ -749,9 +760,9 @@ class SpiderFootCli(cmd.Cmd):
# Show logs.
def do_logs(self, line):
"""logs <sid> [-l count] [-w]
- Show the most recent [count] logs for a given scan ID, <sid>.
+ Show the most recent [count] logs for a given scan ID, <sid>.
If no count is supplied, all logs are given.
- If -w is supplied, logs will be streamed to the console until
+ If -w is supplied, logs will be streamed to the console until
Ctrl-C is entered."""
c = self.myparseline(line)
if len(c[0]) < 1:
@@ -760,11 +771,11 @@ class SpiderFootCli(cmd.Cmd):
sid = c[0][0]
limit = None
if "-l" in c[0]:
- limit = c[0][c[0].index("-l")+1]
+ limit = c[0][c[0].index("-l") + 1]
if "-w" not in c[0]:
d = self.request(self.ownopts['cli.server_baseurl'] + "/scanlog",
- post={ 'id': sid, 'limit': limit })
+ post={'id': sid, 'limit': limit})
if not d:
return
j = json.loads(d)
@@ -772,12 +783,16 @@ class SpiderFootCli(cmd.Cmd):
self.dprint("No results.")
return
- self.send_output(d, line, titles={"0": "Generated",
- "1": "Type",
- "2": "Source",
- "3": "Message"
- }
- )
+ self.send_output(
+ d,
+ line,
+ titles={
+ "0": "Generated",
+ "1": "Type",
+ "2": "Source",
+ "3": "Message"
+ }
+ )
return
else:
# Get the rowid of the latest log message
@@ -796,12 +811,13 @@ class SpiderFootCli(cmd.Cmd):
if not limit:
limit = 10
d = self.request(self.ownopts['cli.server_baseurl'] + "/scanlog",
- post={'id': sid, 'reverse': '1', 'rowId': rowid-int(limit)})
+ post={'id': sid, 'reverse': '1', 'rowId': rowid - int(limit)})
if not d:
return
+
j = json.loads(d)
for r in j:
- #self.send_output(str(r), line, total=False, raw=True)
+ # self.send_output(str(r), line, total=False, raw=True)
if r[2] == "ERROR":
self.edprint(r[1] + ": " + r[3])
else:
@@ -827,13 +843,13 @@ class SpiderFootCli(cmd.Cmd):
# Start a new scan.
def do_start(self, line):
"""start <target> (-m m1,... | -t t1,... | -u case) [-n name] [-w]
- Start a scan against <target> using modules m1,... OR looking
+ Start a scan against <target> using modules m1,... OR looking
for types t1,...
OR by use case ("all", "investigate", "passive" and "footprint").
- Scan be be optionally named [name], without a name the target
+ Scan be be optionally named [name], without a name the target
will be used.
- Use -w to watch the logs from the scan. Ctrl-C to abort the
+ Use -w to watch the logs from the scan. Ctrl-C to abort the
logging (but will not abort the scan).
"""
mods = ""
@@ -845,15 +861,15 @@ class SpiderFootCli(cmd.Cmd):
return
if "-m" in c[0]:
- mods = c[0][c[0].index("-m")+1]
+ mods = c[0][c[0].index("-m") + 1]
if "-t" in c[0]:
# Scan by type
- types = c[0][c[0].index("-t")+1]
+ types = c[0][c[0].index("-t") + 1]
if "-u" in c[0]:
# Scan by use case
- usecase = c[0][c[0].index("-u")+1]
+ usecase = c[0][c[0].index("-u") + 1]
if not mods and not types and not usecase:
self.edprint("Invalid syntax.")
@@ -861,16 +877,17 @@ class SpiderFootCli(cmd.Cmd):
target = c[0][0]
if "-n" in c[0]:
- title = c[0][c[0].index("-n")+1]
+ title = c[0][c[0].index("-n") + 1]
else:
title = target
- post = { "scanname": title,
- "scantarget": target,
- "modulelist": mods,
- "typelist": types,
- "usecase": usecase,
- "cli": "1"
+ post = {
+ "scanname": title,
+ "scantarget": target,
+ "modulelist": mods,
+ "typelist": types,
+ "usecase": usecase,
+ "cli": "1"
}
d = self.request(self.ownopts['cli.server_baseurl'] + "/startscan",
post=post)
@@ -898,8 +915,7 @@ class SpiderFootCli(cmd.Cmd):
self.edprint("Invalid syntax.")
return
- d = self.request(self.ownopts['cli.server_baseurl'] + \
- "/stopscan?id=" + id + "")
+ d = self.request(self.ownopts['cli.server_baseurl'] + "/stopscan?id=" + id)
if not d:
return
@@ -918,7 +934,7 @@ class SpiderFootCli(cmd.Cmd):
# Search for data
def do_find(self, line):
"""find "<string|/regex/>" <[-s sid]|[-t type]> [-x]
- Search for string/regex, limited to the scope of either a scan ID or
+ Search for string/regex, limited to the scope of either a scan ID or
event type. -x for extended format."""
c = self.myparseline(line)
if len(c[0]) < 1:
@@ -928,14 +944,16 @@ class SpiderFootCli(cmd.Cmd):
val = c[0][0]
sid = None
etype = None
+
if "-t" in c[0]:
- etype = c[0][c[0].index("-t")+1]
+ etype = c[0][c[0].index("-t") + 1]
if "-s" in c[0]:
- sid = c[0][c[0].index("-s")+1]
+ sid = c[0][c[0].index("-s") + 1]
- titles={"0": "Last Seen",
- "1": "Data",
- "3": "Module"
+ titles = {
+ "0": "Last Seen",
+ "1": "Data",
+ "3": "Module"
}
if "-x" in c[0]:
titles["2"] = "Source Data"
@@ -958,7 +976,7 @@ class SpiderFootCli(cmd.Cmd):
# Summary of a scan
def do_summary(self, line):
"""summary <sid> [-t]
- Summarise the results for a scan ID, <sid>. -t to only show
+ Summarise the results for a scan ID, <sid>. -t to only show
the element types."""
c = self.myparseline(line)
if len(c[0]) < 1:
@@ -970,15 +988,17 @@ class SpiderFootCli(cmd.Cmd):
if "-t" in c[0]:
titles = {"0": "Element Type"}
else:
- titles={"0": "Element Type",
- "1": "Element Description",
- "3": "Total",
- "4": "Unique"
+ titles = {
+ "0": "Element Type",
+ "1": "Element Description",
+ "3": "Total",
+ "4": "Unique"
}
d = self.request(self.ownopts['cli.server_baseurl'] + "/scansummary?id=" + sid + "&by=type")
if not d:
return
+
j = json.loads(d)
if not j:
@@ -1069,7 +1089,7 @@ class SpiderFootCli(cmd.Cmd):
return
j = list()
serverconfig = dict()
- token = "" # nosec
+ token = "" # nosec
if not d:
self.edprint("Unable to obtain SpiderFoot server-side config.")
else:
@@ -1111,9 +1131,12 @@ class SpiderFootCli(cmd.Cmd):
self.dprint(k + " = " + str(serverconfig[k]), plain=True)
if len(output) > 0:
- self.send_output(json.dumps(output), line,
- { 'opt': "Option", 'val': "Value"},
- total=False)
+ self.send_output(
+ json.dumps(output),
+ line,
+ {'opt': "Option", 'val': "Value"},
+ total=False
+ )
return
if val:
@@ -1184,7 +1207,7 @@ class SpiderFootCli(cmd.Cmd):
"""shell
Run a shell command locally."""
self.dprint("Running shell command:" + str(line))
- self.dprint(os.popen(line).read(), plain=True) # nosec
+ self.dprint(os.popen(line).read(), plain=True) # nosec
def do_clear(self, line):
"""clear
@@ -1204,6 +1227,7 @@ class SpiderFootCli(cmd.Cmd):
print("\n")
return True
+
if __name__ == "__main__":
p = argparse.ArgumentParser(description='SpiderFoot: Open Source Intelligence Automation.')
p.add_argument("-d", "--debug", help="Enable debug output.", action='store_true')
diff --git a/sfdb.py b/sfdb.py
index b230966b..7e106459 100644
--- a/sfdb.py
+++ b/sfdb.py
@@ -16,6 +16,7 @@ import time
import threading
from sflib import SpiderFoot, SpiderFootEvent
+
class SpiderFootDb:
"""SpiderFoot database
@@ -291,7 +292,7 @@ class SpiderFootDb:
# a custom function to do so..
def __dbregex__(qry, data):
try:
- rx = re.compile(qry, re.IGNORECASE|re.DOTALL)
+ rx = re.compile(qry, re.IGNORECASE | re.DOTALL)
ret = rx.match(data)
except BaseException as e:
return False
@@ -319,7 +320,7 @@ class SpiderFootDb:
except BaseException as e:
continue
self.conn.commit()
- #self.conn.close()
+ # self.conn.close()
#
# Back-end database operations
@@ -397,7 +398,7 @@ class SpiderFootDb:
raise ValueError(f"No valid search criteria provided; expected: {', '.join(valid_criteria)}")
if len(criteria) == 1:
- raise ValueError(f"Only one search criteria provided; expected at least two")
+ raise ValueError("Only one search criteria provided; expected at least two")
qvars = list()
qry = "SELECT ROUND(c.generated) AS generated, c.data, \
@@ -434,8 +435,6 @@ class SpiderFootDb:
with self.dbhLock:
try:
- #print(qry)
- #print(str(qvars))
self.dbh.execute(qry, qvars)
return self.dbh.fetchall()
except sqlite3.Error as e:
@@ -473,7 +472,7 @@ class SpiderFootDb:
Raises:
IOError: database I/O failed
-
+
Todo:
Do something smarter to handle database locks
"""
@@ -502,8 +501,8 @@ class SpiderFootDb:
self.conn.commit()
except sqlite3.Error as e:
if "locked" in e.args[0] or "thread" in e.args[0]:
- #print("[warning] Couldn't log due to SQLite limitations. You can probably ignore this.")
- #self.sf.fatal(f"Unable to log event in DB due to lock: {e.args[0]}")
+ # print("[warning] Couldn't log due to SQLite limitations. You can probably ignore this.")
+ # self.sf.fatal(f"Unable to log event in DB due to lock: {e.args[0]}")
pass
else:
raise IOError(f"Unable to log scan event in DB: {e.args[0]}")
@@ -697,7 +696,7 @@ class SpiderFootDb:
if not isinstance(eventType, str):
raise TypeError(f"eventType is {type(eventType)}; expected str()")
-
+
qry = "SELECT ROUND(c.generated) AS generated, c.data, \
s.data as 'source_data', \
c.module, c.type, c.confidence, c.visibility, c.risk, c.hash, \
@@ -747,7 +746,7 @@ class SpiderFootDb:
if not isinstance(eventType, str):
raise TypeError(f"eventType is {type(eventType)}; expected str()")
-
+
qry = "SELECT DISTINCT data, type, COUNT(*) FROM tbl_scan_results \
WHERE scan_instance_id = ?"
qvars = [instanceId]
@@ -940,7 +939,7 @@ class SpiderFootDb:
raise TypeError(f"optMap is {type(optMap)}; expected dict()")
if not optMap:
raise ValueError("optMap is empty")
-
+
qry = "REPLACE INTO tbl_config (scope, opt, val) VALUES (?, ?, ?)"
with self.dbhLock:
@@ -974,7 +973,7 @@ class SpiderFootDb:
"""
qry = "SELECT scope, opt, val FROM tbl_config"
-
+
retval = dict()
with self.dbhLock:
@@ -1029,7 +1028,7 @@ class SpiderFootDb:
raise TypeError(f"optMap is {type(optMap)}; expected dict()")
if not optMap:
raise ValueError("optMap is empty")
-
+
qry = "REPLACE INTO tbl_scan_config \
(scan_instance_id, component, opt, val) VALUES (?, ?, ?, ?)"
@@ -1458,7 +1457,7 @@ class SpiderFootDb:
while keepGoing:
nextSet = self.scanElementChildrenDirect(instanceId, nextIds)
- if nextSet == None or len(nextSet) == 0:
+ if nextSet is None or len(nextSet) == 0:
keepGoing = False
break
diff --git a/sflib.py b/sflib.py
index d15e6cb7..a0fbd056 100644
--- a/sflib.py
+++ b/sflib.py
@@ -16,6 +16,7 @@ from stem.control import Controller
import inspect
import hashlib
import html
+import io
import urllib.request, urllib.parse, urllib.error
import json
import re
@@ -27,21 +28,18 @@ import ssl
import sys
import time
import netaddr
-import urllib.request, urllib.error, urllib.parse
-import threading
import traceback
import OpenSSL
import uuid
import cryptography
import dns.resolver
+import urllib3
from publicsuffixlist import PublicSuffixList
from networkx import nx
from networkx.readwrite.gexf import GEXFWriter
from datetime import datetime
from bs4 import BeautifulSoup, SoupStrainer
from copy import deepcopy
-import io
-import urllib3
# For hiding the SSL warnings coming from the requests lib
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
@@ -56,7 +54,7 @@ class SpiderFoot:
socksProxy (str): SOCKS proxy
opts (dict): configuration options
"""
-
+
_dbh = None
_scanId = None
_socksProxy = None
@@ -175,9 +173,9 @@ class SpiderFoot:
f.close()
return ret
- except BaseException as b:
+ except BaseException as e:
if fatal:
- self.error("Unable to open option file, " + fname + ".")
+ self.error(f"Unable to open option file, {fname}.")
else:
return None
@@ -245,15 +243,15 @@ class SpiderFoot:
for [parent, id] in parents[entity]:
if parent in entities:
if entity != parent:
- #print("Adding entity parent: " + parent)
+ # print("Adding entity parent: " + parent)
mapping.add((entity, parent))
else:
ppids = list()
- #print("Checking " + parent + " for entityship.")
+ # print("Checking " + parent + " for entityship.")
next_parents = get_next_parent_entities(parent, ppids)
for next_parent in next_parents:
if entity != next_parent:
- #print("Adding next entity parent: " + next_parent)
+ # print("Adding next entity parent: " + next_parent)
mapping.add((entity, next_parent))
return mapping
@@ -293,7 +291,7 @@ class SpiderFoot:
if dst in root:
col = ["255", "0", "0"]
node = graph.add_node(dst)
- graph.node[dst]['viz'] = {'color': { 'r': col[0], 'g': col[1], 'b': col[2] } }
+ graph.node[dst]['viz'] = {'color': {'r': col[0], 'g': col[1], 'b': col[2]}}
nodelist[dst] = ncounter
if src not in nodelist:
@@ -301,7 +299,7 @@ class SpiderFoot:
if src in root:
col = ["255", "0", "0"]
graph.add_node(src)
- graph.node[src]['viz'] = {'color': { 'r': col[0], 'g': col[1], 'b': col[2] } }
+ graph.node[src]['viz'] = {'color': {'r': col[0], 'g': col[1], 'b': col[2]}}
nodelist[src] = ncounter
graph.add_edge(src, dst)
@@ -336,36 +334,47 @@ class SpiderFoot:
# Leave out this special case
if dst == "ROOT" or src == "ROOT":
continue
+
if dst not in nodelist:
ncounter = ncounter + 1
+
if dst in root:
col = "#f00"
- ret['nodes'].append({'id': str(ncounter),
- 'label': str(dst),
- 'x': random.SystemRandom().randint(1, 1000),
- 'y': random.SystemRandom().randint(1, 1000),
- 'size': "1",
- 'color': col
+
+ ret['nodes'].append({
+ 'id': str(ncounter),
+ 'label': str(dst),
+ 'x': random.SystemRandom().randint(1, 1000),
+ 'y': random.SystemRandom().randint(1, 1000),
+ 'size': "1",
+ 'color': col
})
+
nodelist[dst] = ncounter
if src not in nodelist:
+ ncounter = ncounter + 1
+
if src in root:
col = "#f00"
- ncounter = ncounter + 1
- ret['nodes'].append({'id': str(ncounter),
- 'label': str(src),
- 'x': random.SystemRandom().randint(1, 1000),
- 'y': random.SystemRandom().randint(1, 1000),
- 'size': "1",
- 'color': col
+
+ ret['nodes'].append({
+ 'id': str(ncounter),
+ 'label': str(src),
+ 'x': random.SystemRandom().randint(1, 1000),
+ 'y': random.SystemRandom().randint(1, 1000),
+ 'size': "1",
+ 'color': col
})
+
nodelist[src] = ncounter
ecounter = ecounter + 1
- ret['edges'].append({'id': str(ecounter),
- 'source': str(nodelist[src]),
- 'target': str(nodelist[dst])
+
+ ret['edges'].append({
+ 'id': str(ecounter),
+ 'source': str(nodelist[src]),
+ 'target': str(nodelist[dst])
})
return json.dumps(ret)
@@ -810,7 +819,7 @@ class SpiderFoot:
# Parse the target and set the targetType
for rxpair in regexToType:
rx = list(rxpair.keys())[0]
- if re.match(rx, target, re.IGNORECASE|re.UNICODE):
+ if re.match(rx, target, re.IGNORECASE | re.UNICODE):
targetType = list(rxpair.values())[0]
break
return targetType
@@ -1147,7 +1156,7 @@ class SpiderFoot:
ps = PublicSuffixList(tldList, only_icann=True, accept_unknown=False)
sfx = ps.privatesuffix(hostname)
return sfx == hostname
-
+
def validIP(self, address):
"""Check if the provided string is a valid IPv4 address.
@@ -1264,8 +1273,8 @@ class SpiderFoot:
"""
chars = ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm',
- 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z',
- '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '-', '.']
+ 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z',
+ '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '-', '.']
for c in cmd:
if c.lower() not in chars:
return False
@@ -1290,14 +1299,14 @@ class SpiderFoot:
wd = dict()
- dicts = [ "english", "german", "french", "spanish" ]
+ dicts = ["english", "german", "french", "spanish"]
for d in dicts:
try:
with io.open(self.myPath() + "/dicts/ispell/" + d + ".dict", 'r', encoding='utf8', errors='ignore') as wdct:
dlines = wdct.readlines()
except BaseException as e:
- self.debug("Could not read dictionary: " + str(e))
+ self.debug(f"Could not read dictionary: {e}")
continue
for w in dlines:
@@ -1315,7 +1324,7 @@ class SpiderFoot:
wd = dict()
- dicts = [ "names" ]
+ dicts = ["names"]
for d in dicts:
try:
@@ -1332,7 +1341,6 @@ class SpiderFoot:
return list(wd.keys())
-
def dataParentChildToTree(self, data):
"""Converts a dictionary of k -> array to a nested
tree that can be digested by d3 for visualizations.
@@ -1500,7 +1508,7 @@ class SpiderFoot:
t = target.targetType
v = target.targetValue
- if t in [ "IP_ADDRESS", "IPV6_ADDRESS" ]:
+ if t in ["IP_ADDRESS", "IPV6_ADDRESS"]:
r = self.resolveIP(v)
if r:
ret.extend(r)
@@ -1625,7 +1633,7 @@ class SpiderFoot:
emails.add(match)
return list(emails)
-
+
def parseCreditCards(self, data):
"""Find all credit card numbers with the supplied content.
@@ -1644,13 +1652,13 @@ class SpiderFoot:
if not isinstance(data, str):
return list()
- creditCards = set()
+ creditCards = set()
- # Remove whitespace from data.
- # Credit cards might contain spaces between them
+ # Remove whitespace from data.
+ # Credit cards might contain spaces between them
# which will cause regex mismatch
data = data.replace(" ", "")
-
+
# Extract all numbers with lengths ranging from 13 - 19 digits
matches = re.findall(r"[0-9]{13,19}", data)
@@ -1680,7 +1688,7 @@ class SpiderFoot:
else:
self.debug("Skipped invalid credit card number: " + match)
return list(creditCards)
-
+
def parseIBANNumbers(self, data):
"""Find all International Bank Account Numbers (IBANs) within the supplied content.
@@ -1703,30 +1711,30 @@ class SpiderFoot:
# Dictionary of country codes and their respective IBAN lengths
ibanCountryLengths = {
- "AL" : 28, "AD" : 24, "AT" : 20, "AZ" : 28,
- "ME" : 22, "BH" : 22, "BY" : 28, "BE" : 16,
- "BA" : 20, "BR" : 29, "BG" : 22, "CR" : 22,
- "HR" : 21, "CY" : 28, "CZ" : 24, "DK" : 18,
- "DO" : 28, "EG" : 29, "SV" : 28, "FO" : 18,
- "FI" : 18, "FR" : 27, "GE" : 22, "DE" : 22,
- "GI" : 23, "GR" : 27, "GL" : 18, "GT" : 28,
- "VA" : 22, "HU" : 28, "IS" : 26, "IQ" : 23,
- "IE" : 22, "IL" : 23, "JO" : 30, "KZ" : 20,
- "XK" : 20, "KW" : 30, "LV" : 21, "LB" : 28,
- "LI" : 21, "LT" : 20, "LU" : 20, "MT" : 31,
- "MR" : 27, "MU" : 30, "MD" : 24, "MC" : 27,
- "DZ" : 24, "AO" : 25, "BJ" : 28, "VG" : 24,
- "BF" : 27, "BI" : 16, "CM" : 27, "CV" : 25,
- "CG" : 27, "EE" : 20, "GA" : 27, "GG" : 22,
- "IR" : 26, "IM" : 22, "IT" : 27, "CI" : 28,
- "JE" : 22, "MK" : 19, "MG" : 27, "ML" : 28,
- "MZ" : 25, "NL" : 18, "NO" : 15, "PK" : 24,
- "PS" : 29, "PL" : 28, "PT" : 25, "QA" : 29,
- "RO" : 24, "LC" : 32, "SM" : 27, "ST" : 25,
- "SA" : 24, "SN" : 28, "RS" : 22, "SC" : 31,
- "SK" : 24, "SI" : 19, "ES" : 24, "CH" : 21,
- "TL" : 23, "TN" : 24, "TR" : 26, "UA" : 29,
- "AE" : 23, "GB" : 22, "SE" : 24
+ "AL": 28, "AD": 24, "AT": 20, "AZ": 28,
+ "ME": 22, "BH": 22, "BY": 28, "BE": 16,
+ "BA": 20, "BR": 29, "BG": 22, "CR": 22,
+ "HR": 21, "CY": 28, "CZ": 24, "DK": 18,
+ "DO": 28, "EG": 29, "SV": 28, "FO": 18,
+ "FI": 18, "FR": 27, "GE": 22, "DE": 22,
+ "GI": 23, "GR": 27, "GL": 18, "GT": 28,
+ "VA": 22, "HU": 28, "IS": 26, "IQ": 23,
+ "IE": 22, "IL": 23, "JO": 30, "KZ": 20,
+ "XK": 20, "KW": 30, "LV": 21, "LB": 28,
+ "LI": 21, "LT": 20, "LU": 20, "MT": 31,
+ "MR": 27, "MU": 30, "MD": 24, "MC": 27,
+ "DZ": 24, "AO": 25, "BJ": 28, "VG": 24,
+ "BF": 27, "BI": 16, "CM": 27, "CV": 25,
+ "CG": 27, "EE": 20, "GA": 27, "GG": 22,
+ "IR": 26, "IM": 22, "IT": 27, "CI": 28,
+ "JE": 22, "MK": 19, "MG": 27, "ML": 28,
+ "MZ": 25, "NL": 18, "NO": 15, "PK": 24,
+ "PS": 29, "PL": 28, "PT": 25, "QA": 29,
+ "RO": 24, "LC": 32, "SM": 27, "ST": 25,
+ "SA": 24, "SN": 28, "RS": 22, "SC": 31,
+ "SK": 24, "SI": 19, "ES": 24, "CH": 21,
+ "TL": 23, "TN": 24, "TR": 26, "UA": 29,
+ "AE": 23, "GB": 22, "SE": 24
}
# Normalize input data to remove whitespace
@@ -1736,7 +1744,7 @@ class SpiderFoot:
# and starting with two characters
matches = re.findall("[A-Za-z]{2}[A-Za-z0-9]{13,30}", data)
- for match in matches:
+ for match in matches:
iban = match.upper()
countryCode = iban[0:2]
@@ -1744,7 +1752,7 @@ class SpiderFoot:
if countryCode not in ibanCountryLengths.keys():
self.debug("Skipped invalid IBAN (invalid country code): %s" % iban)
continue
-
+
if len(iban) != ibanCountryLengths[countryCode]:
self.debug("Skipped invalid IBAN (invalid length): %s" % iban)
continue
@@ -1830,7 +1838,7 @@ class SpiderFoot:
if ret['expiry'] <= now:
ret['expired'] = True
except BaseException as e:
- self.error("Error processing date in certificate: " + str(e) , False)
+ self.error(f"Error processing date in certificate: {e}", False)
ret['certerror'] = True
return ret
@@ -1952,8 +1960,7 @@ class SpiderFoot:
try:
for t in list(tags.keys()):
- for lnk in BeautifulSoup(data, "lxml",
- parse_only=SoupStrainer(t)).find_all(t):
+ for lnk in BeautifulSoup(data, "lxml", parse_only=SoupStrainer(t)).find_all(t):
if lnk.has_attr(tags[t]):
urlsRel.append(lnk[tags[t]])
except BaseException as e:
@@ -1973,10 +1980,10 @@ class SpiderFoot:
# Don't include stuff likely part of some dynamically built incomplete
# URL found in Javascript code (character is part of some logic)
- if link[len(link) - 1] == '.' or link[0] == '+' or \
- 'javascript:' in linkl or '()' in link:
+ if link[len(link) - 1] == '.' or link[0] == '+' or 'javascript:' in linkl or '()' in link:
self.debug('unlikely link: ' + link)
continue
+
# Filter in-page links
if re.match('.*#.[^/]+', link):
self.debug('in-page link: ' + link)
@@ -2068,8 +2075,28 @@ class SpiderFoot:
proxies = dict()
if self.opts['_socks1type']:
- neverProxyNames = [ self.opts['_socks2addr'] ]
- neverProxySubnets = [ "192.168.", "127.", "10." ]
+ neverProxyNames = [self.opts['_socks2addr']]
+ neverProxySubnets = [
+ "127."
+ "10.",
+ "192.168.",
+ "172.16.",
+ "172.17.",
+ "172.18.",
+ "172.19.",
+ "172.20.",
+ "172.21.",
+ "172.22.",
+ "172.23.",
+ "172.24.",
+ "172.25.",
+ "172.26.",
+ "172.27.",
+ "172.28.",
+ "172.29.",
+ "172.30.",
+ "172.31."
+ ]
host = self.urlFQDN(url)
# Completely on or off?
@@ -2118,12 +2145,15 @@ class SpiderFoot:
if sizeLimit or headOnly:
if not noLog:
- self.info("Fetching (HEAD only): " + self.removeUrlCreds(url) + \
- " [user-agent: " + header['User-Agent'] + "] [timeout: " + \
- str(timeout) + "]")
-
- hdr = self.getSession().head(url, headers=header, proxies=proxies,
- verify=verify, timeout=timeout)
+ self.info(f"Fetching (HEAD only): {self.removeUrlCreds(url)} [user-agent: {header['User-Agent']}] [timeout: {timeout}]")
+
+ hdr = self.getSession().head(
+ url,
+ headers=header,
+ proxies=proxies,
+ verify=verify,
+ timeout=timeout
+ )
size = int(hdr.headers.get('content-length', 0))
newloc = hdr.headers.get('location', url).strip()
@@ -2141,12 +2171,15 @@ class SpiderFoot:
if result['realurl'] != url:
if not noLog:
- self.info("Fetching (HEAD only): " + self.removeUrlCreds(url) + \
- " [user-agent: " + header['User-Agent'] + "] [timeout: " + \
- str(timeout) + "]")
-
- hdr = self.getSession().head(result['realurl'], headers=header, proxies=proxies,
- verify=verify, timeout=timeout)
+ self.info(f"Fetching (HEAD only): {self.removeUrlCreds(url)} [user-agent: {header['User-Agent']}] [timeout: {timeout}]")
+
+ hdr = self.getSession().head(
+ result['realurl'],
+ headers=header,
+ proxies=proxies,
+ verify=verify,
+ timeout=timeout
+ )
size = int(hdr.headers.get('content-length', 0))
result['realurl'] = hdr.headers.get('location', result['realurl'])
result['code'] = str(hdr.status_code)
@@ -2156,21 +2189,32 @@ class SpiderFoot:
if not noLog:
if cookies is None:
- self.info("Fetching: " + self.removeUrlCreds(url) + " [user-agent: " + \
- header['User-Agent'] + "] [timeout: " + str(timeout) + "]")
+ self.info(f"Fetching: {self.removeUrlCreds(url)} [user-agent: {header['User-Agent']}] [timeout: {timeout}]")
else:
- self.info("Fetching (incl. cookies): " + self.removeUrlCreds(url) + \
- " [user-agent: " + header['User-Agent'] + "] [timeout: " + \
- str(timeout) + "]")
+ self.info(f"Fetching (incl. cookies): {self.removeUrlCreds(url)} [user-agent: {header['User-Agent']}] [timeout: {timeout}]")
try:
if postData:
- res = self.getSession().post(url, data=postData, headers=header, proxies=proxies,
- allow_redirects=True, cookies=cookies,
- timeout=timeout, verify=verify)
+ res = self.getSession().post(
+ url,
+ data=postData,
+ headers=header,
+ proxies=proxies,
+ allow_redirects=True,
+ cookies=cookies,
+ timeout=timeout,
+ verify=verify
+ )
else:
- res = self.getSession().get(url, headers=header, proxies=proxies, allow_redirects=True,
- cookies=cookies, timeout=timeout, verify=verify)
+ res = self.getSession().get(
+ url,
+ headers=header,
+ proxies=proxies,
+ allow_redirects=True,
+ cookies=cookies,
+ timeout=timeout,
+ verify=verify
+ )
except requests.exceptions.RequestException:
raise Exception('Failed to connect to %s' % url) from None
@@ -2234,7 +2278,7 @@ class SpiderFoot:
m = mod.__name__
atime = time.time()
t = str(atime - btime)
- self.info("Fetched data: " + str(len(result['content'] or '')) + \
+ self.info("Fetched data: " + str(len(result['content'] or '')) +
" (" + self.removeUrlCreds(url) + "), took " + t + "s")
return result
@@ -2421,7 +2465,7 @@ class SpiderFootPlugin(object):
_priority = 1
# Error state of the module
errorState = False
-
+
def __init__(self):
"""Not really needed in most cases."""
pass
@@ -2542,13 +2586,16 @@ class SpiderFootPlugin(object):
if self.__outputFilter__:
# Be strict about what events to pass on, unless they are
# the ROOT event or the event type of the target.
- if eventName != 'ROOT' and eventName != self.getTarget().targetType \
- and eventName not in self.__outputFilter__:
- return None
+ if eventName != 'ROOT' and eventName != self.getTarget().targetType:
+ if eventName not in self.__outputFilter__:
+ return None
storeOnly = False # Under some conditions, only store and don't notify
- if sfEvent.data is None or (type(sfEvent.data) is str and len(sfEvent.data) == 0):
+ if sfEvent.data is None:
+ return None
+
+ if type(sfEvent.data) is str and len(sfEvent.data) == 0:
return None
if self.checkForStop():
@@ -2571,8 +2618,7 @@ class SpiderFootPlugin(object):
prevEvent = sfEvent.sourceEvent
while prevEvent is not None:
if prevEvent.sourceEvent is not None:
- if prevEvent.sourceEvent.eventType == sfEvent.eventType and \
- prevEvent.sourceEvent.data.lower() == sfEvent.data.lower():
+ if prevEvent.sourceEvent.eventType == sfEvent.eventType and prevEvent.sourceEvent.data.lower() == sfEvent.data.lower():
storeOnly = True
break
prevEvent = prevEvent.sourceEvent
@@ -2715,7 +2761,7 @@ class SpiderFootTarget(object):
def targetType(self, targetType):
if not isinstance(targetType, str):
raise TypeError(f"targetType is {type(targetType)}; expected str()")
-
+
if targetType not in self._validTypes:
raise ValueError(f"targetType value is {targetType}; expected {self._validTypes}")
@@ -2760,7 +2806,7 @@ class SpiderFootTarget(object):
if value is None:
return
-
+
if {'type': typeName, 'value': value} in self.targetAliases:
return None
@@ -2851,7 +2897,7 @@ class SpiderFootTarget(object):
# We can't really say anything about names, username or phone numbers,
# so everything matches
- if self.targetType in ["HUMAN_NAME", "PHONE_NUMBER", "USERNAME" ]:
+ if self.targetType in ["HUMAN_NAME", "PHONE_NUMBER", "USERNAME"]:
return True
if netaddr.valid_ipv4(value):
@@ -2862,9 +2908,8 @@ class SpiderFootTarget(object):
if self.targetType == "NETBLOCK_OWNER":
if netaddr.IPAddress(value) in netaddr.IPNetwork(self.targetValue):
return True
- if self.targetType in [ "IP_ADDRESS", "IPV6_ADDRESS" ]:
- if netaddr.IPAddress(value) in \
- netaddr.IPNetwork(netaddr.IPAddress(self.targetValue)):
+ if self.targetType in ["IP_ADDRESS", "IPV6_ADDRESS"]:
+ if netaddr.IPAddress(value) in netaddr.IPNetwork(netaddr.IPAddress(self.targetValue)):
return True
else:
for name in self.getNames():
@@ -3025,7 +3070,7 @@ class SpiderFootEvent(object):
TypeError: confidence type was invalid
ValueError: confidence value was invalid
"""
-
+
if not isinstance(eventType, str):
raise TypeError(f"eventType is {type(eventType)}; expected str()")
@@ -3044,7 +3089,7 @@ class SpiderFootEvent(object):
TypeError: confidence type was invalid
ValueError: confidence value was invalid
"""
-
+
if not isinstance(confidence, int):
raise TypeError(f"confidence is {type(confidence)}; expected int()")
@@ -3063,7 +3108,7 @@ class SpiderFootEvent(object):
TypeError: visibility type was invalid
ValueError: visibility value was invalid
"""
-
+
if not isinstance(visibility, int):
raise TypeError(f"visibility is {type(visibility)}; expected int()")
@@ -3082,7 +3127,7 @@ class SpiderFootEvent(object):
TypeError: risk type was invalid
ValueError: risk value was invalid
"""
-
+
if not isinstance(risk, int):
raise TypeError(f"risk is {type(risk)}; expected int()")
@@ -3098,7 +3143,7 @@ class SpiderFootEvent(object):
TypeError: module type was invalid
ValueError: module value was invalid
"""
-
+
if not isinstance(module, str):
raise TypeError(f"module is {type(module )}; expected str()")
@@ -3115,7 +3160,7 @@ class SpiderFootEvent(object):
TypeError: data type was invalid
ValueError: data value was invalid
"""
-
+
if not isinstance(data, str):
raise TypeError(f"data is {type(data)}; expected str()")
@@ -3130,7 +3175,7 @@ class SpiderFootEvent(object):
Raises:
TypeError: sourceEvent type was invalid
"""
-
+
# "ROOT" is a special "hash" reserved for elements with no parent,
# such as targets provided via the web UI or CLI.
if self.eventType == "ROOT":
diff --git a/sfscan.py b/sfscan.py
index b8274ff0..f6778aaa 100644
--- a/sfscan.py
+++ b/sfscan.py
@@ -14,11 +14,11 @@ import time
import sys
import socket
import dns.resolver
-import random
from copy import deepcopy
from sfdb import SpiderFootDb
from sflib import SpiderFoot, SpiderFootEvent, SpiderFootTarget, SpiderFootPlugin
+
class SpiderFootScanner():
"""SpiderFootScanner object.
@@ -337,9 +337,9 @@ class SpiderFootScanner():
self.__setStatus("FINISHED", None, time.time() * 1000)
except BaseException as e:
exc_type, exc_value, exc_traceback = sys.exc_info()
- self.__sf.error("Unhandled exception (" + e.__class__.__name__ + ") " + \
- "encountered during scan. Please report this as a bug: " + \
- repr(traceback.format_exception(exc_type, exc_value, exc_traceback)), False)
+ self.__sf.error("Unhandled exception (" + e.__class__.__name__ + ") " +
+ "encountered during scan. Please report this as a bug: " +
+ repr(traceback.format_exception(exc_type, exc_value, exc_traceback)), False)
self.__sf.status("Scan [" + self.__scanId + "] failed: " + str(e))
self.__setStatus("ERROR-FAILED", None, time.time() * 1000)