SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.
Python attacksurfacecticybersecurityfootprintinghackinginformation-gatheringinformation-securityinfosecintelligence-gatheringosintosint-frameworkosint-reconnaissanceosint-toolpentestingpythonreconsecurity-toolsthreat-intelligencethreatintel
Update style for pep8
Brendan Colesccdb1e525f1c
diff --git a/sf.py b/sf.pyindex 902a7681..12a46e08 100755--- a/sf.py+++ b/sf.py@@ -37,8 +37,8 @@ from sfscan import SpiderFootScanner# be overridden from saved configuration settings stored in the DB.sfConfig = {'_debug': False, # Debug- '__logging': True, # Logging in general- '__outputfilter': None, # Event types to filter from modules' output+ '__logging': True, # Logging in general+ '__outputfilter': None, # Event types to filter from modules' output'__blocknotif': False, # Block notifications'_fatalerrors': False,'_useragent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:62.0) Gecko/20100101 Firefox/62.0', # User-Agent to use for HTTP requests@@ -85,6 +85,7 @@ sfOptdescs = {scanId = Nonedbh = None+def handle_abort(signal, frame):"""handle interrupt and abort scan."""print("[*] Aborting...")@@ -92,6 +93,7 @@ def handle_abort(signal, frame):dbh.scanInstanceSet(scanId, None, None, "ABORTED")sys.exit(-1)+if __name__ == '__main__':if len(sys.argv) == 0:print("SpiderFoot 3.x now requires -l <ip>:<port> to start the web server.")@@ -358,7 +360,7 @@ if __name__ == '__main__':if args.D:delim = args.Delse:- if args.o in [ "tab", None ]:+ if args.o in ["tab", None]:delim = "\t"if args.o == "csv":@@ -380,7 +382,7 @@ if __name__ == '__main__':if not info:time.sleep(1)continue- if info[5] in [ "ERROR-FAILED", "ABORT-REQUESTED", "ABORTED", "FINISHED" ]:+ if info[5] in ["ERROR-FAILED", "ABORT-REQUESTED", "ABORTED", "FINISHED"]:if sfConfig['__logging']:print("[*] Scan completed with status " + info[5])if args.o == "json":@@ -451,11 +453,11 @@ if __name__ == '__main__':print("Warning: passwd file contains no passwords. Authentication disabled.")print("********************************************************************")else:- print("")- print("********************************************************************")- print("Please consider adding authentication to protect this instance!")- print("Refer to https://www.spiderfoot.net/documentation/#security.")- print("********************************************************************")+ print("")+ print("********************************************************************")+ print("Please consider adding authentication to protect this instance!")+ print("Refer to https://www.spiderfoot.net/documentation/#security.")+ print("********************************************************************")key_path = sf.dataPath() + '/spiderfoot.key'crt_path = sf.dataPath() + '/spiderfoot.crt'diff --git a/sfcli.py b/sfcli.pyindex ac607b3b..77b16616 100755--- a/sfcli.py+++ b/sfcli.py@@ -28,6 +28,7 @@ try:except:import pyreadline as readline+# Colors to make things purtyclass bcolors:GREYBLUE = '\x1b[38;5;25m'@@ -38,6 +39,7 @@ class bcolors:ENDC = '\033[0m'GREYBLUE_DARK = '\x1b[38;5;24m'+class SpiderFootCli(cmd.Cmd):version = "3.2-DEV"pipecmd = None@@ -125,8 +127,7 @@ class SpiderFootCli(cmd.Cmd):if err or not self.ownopts["cli.silent"]:if not plain or color:- cout = col + bcolors.BOLD + pfx + " " + bcolors.ENDC + \- col + msg + bcolors.ENDC+ cout = col + bcolors.BOLD + pfx + " " + bcolors.ENDC + col + msg + bcolors.ENDC# Never include color in the spoolsout = pfx + " " + msgelse:@@ -165,7 +166,7 @@ class SpiderFootCli(cmd.Cmd):def do_history(self, line):"""history [-l]- Short-cut command for set cli.history = 1/0.+ Short-cut command for set cli.history = 1/0.Add -l to just list the history."""c = self.myparseline(line)@@ -242,12 +243,11 @@ class SpiderFootCli(cmd.Cmd):# we have a dict keycn = cv = str(r[c])- #print(str(cn) + ", " + str(c) + ", " + str(v))+ # print(str(cn) + ", " + str(c) + ", " + str(v))if len(v) > maxsize.get(cn, 0):maxsize[cn] = len(v)i += 1-# Adjust for long titlesif titlemap:for c in maxsize:@@ -261,32 +261,32 @@ class SpiderFootCli(cmd.Cmd):t = titlemap.get(c, c)else:t = c- #out += t+ # out += tout.append(t)sdiff = maxsize[c] - len(t) + 1- #out += " " * spaces+ # out += " " * spacesout.append(" " * spaces)- if sdiff > 0 and i < len(cols)-1:- #out += " " * sdiff+ if sdiff > 0 and i < len(cols) - 1:+ # out += " " * sdiffout.append(" " * sdiff)i += 1- #out += "\n"+ # out += "\n"out.append('\n')# Then the separatori = 0for c in cols:- #out += "-" * ((maxsize[c]+spaces))- out.append("-" * ((maxsize[c]+spaces)))- if i < len(cols)-1:- #out += "+"+ # out += "-" * ((maxsize[c]+spaces))+ out.append("-" * ((maxsize[c] + spaces)))+ if i < len(cols) - 1:+ # out += "+"out.append("+")i += 1- #out += "\n"+ # out += "\n"out.append("\n")# Then the actual data- #ts = time.time()+ # ts = time.time()for r in data:i = 0di = 0@@ -317,19 +317,18 @@ class SpiderFootCli(cmd.Cmd):sdiff = (maxsize[cn] - lv) + spaceselse:sdiff = (maxsize[cn] - lv) + spaces - 1- if di < len(cols)-1:- #out += " " * sdiff+ if di < len(cols) - 1:+ # out += " " * sdiffout.append(" " * sdiff)- if di < len(cols)-1:- #out += "| "+ if di < len(cols) - 1:+ # out += "| "out.append("| ")di += 1i += 1- #out += "\n"+ # out += "\n"out.append("\n")- #print("time: " + str(time.time() - ts))- #return ""+ # print("time: " + str(time.time() - ts))return ''.join(out)# Make a request to the SpiderFoot server@@ -342,11 +341,11 @@ class SpiderFootCli(cmd.Cmd):self.edprint("Invalid request URL: %s" % url)return None- #logging.basicConfig()- #logging.getLogger().setLevel(logging.DEBUG)- #requests_log = logging.getLogger("requests.packages.urllib3")- #requests_log.setLevel(logging.DEBUG)- #requests_log.propagate = True+ # logging.basicConfig()+ # logging.getLogger().setLevel(logging.DEBUG)+ # requests_log = logging.getLogger("requests.packages.urllib3")+ # requests_log.setLevel(logging.DEBUG)+ # requests_log.propagate = Truetry:headers = {"User-agent": "SpiderFoot-CLI/" + self.version,@@ -356,26 +355,28 @@ class SpiderFootCli(cmd.Cmd):self.ddprint("Fetching: " + url)self.ddprint("Posting: " + str(post))if not post:- r = requests.get(url,- headers = headers,- verify = self.ownopts['cli.ssl_verify'],- auth = requests.auth.HTTPDigestAuth(- self.ownopts['cli.username'],- self.ownopts['cli.password']- )+ r = requests.get(+ url,+ headers=headers,+ verify=self.ownopts['cli.ssl_verify'],+ auth=requests.auth.HTTPDigestAuth(+ self.ownopts['cli.username'],+ self.ownopts['cli.password'])+ )else:- r = requests.post(url,- headers = headers,- verify = self.ownopts['cli.ssl_verify'],- auth = requests.auth.HTTPDigestAuth(- self.ownopts['cli.username'],- self.ownopts['cli.password']- ),- data = post- )+ r = requests.post(+ url,+ headers=headers,+ verify=self.ownopts['cli.ssl_verify'],+ auth=requests.auth.HTTPDigestAuth(+ self.ownopts['cli.username'],+ self.ownopts['cli.password']+ ),+ data=post+ )self.ddprint("Response: " + str(r))- if r.status_code == requests.codes.ok: #pylint: disable=no-member+ if r.status_code == requests.codes.ok: # pylint: disable=no-memberreturn r.textelse:r.raise_for_status()@@ -410,7 +411,7 @@ class SpiderFootCli(cmd.Cmd):ret[1] = list()i = 0ret[1].append(list())- for t in s[(s.index('|')+1):]:+ for t in s[(s.index('|') + 1):]:if t == '|':i += 1ret[1].append(list())@@ -467,7 +468,7 @@ class SpiderFootCli(cmd.Cmd):returnpipecmd = pc[0]pipeargs = " ".join(pc[1:])- if pipecmd not in [ "str", "regex", "file", "grep", "top", "last" ]:+ if pipecmd not in ["str", "regex", "file", "grep", "top", "last"]:self.edprint("Unrecognised pipe command.")return@@ -548,7 +549,7 @@ class SpiderFootCli(cmd.Cmd):if s[1] != self.version:self.edprint("Server and CLI version are not the same (" + s[1] + \- " / " + self.version + "). This could lead to unpredictable results!")+ " / " + self.version + "). This could lead to unpredictable results!")# List all SpiderFoot modules.def do_modules(self, line, cacheonly=False):@@ -570,15 +571,24 @@ class SpiderFootCli(cmd.Cmd):"""typesList all available element types and their descriptions."""d = self.request(self.ownopts['cli.server_baseurl'] + "/eventtypes")+if not d:return+if cacheonly:j = json.loads(d)for t in j:self.types.append(t[0])return- self.send_output(d, line, titles={ "1": "Element description",- "0": "Element name"})++ self.send_output(+ d,+ line,+ titles={+ "1": "Element description",+ "0": "Element name"+ }+ )# Load commands from a file.def do_load(self, line):@@ -589,7 +599,7 @@ class SpiderFootCli(cmd.Cmd):# Get scan info and config.def do_scaninfo(self, line):"""scaninfo <sid> [-c]- Get status information for scan ID <sid>, optionally also its+ Get status information for scan ID <sid>, optionally also itsconfiguration if -c is supplied."""c = self.myparseline(line)if len(c[0]) < 1:@@ -605,7 +615,7 @@ class SpiderFootCli(cmd.Cmd):self.dprint("No such scan exists.")return- out = "Name: " + j['meta'][0] + "\n"+ out = "Name: " + j['meta'][0] + "\n"out += "ID: " + sid + "\n"out += "Target: " + j['meta'][1] + "\n"out += "Started: " + j['meta'][3] + "\n"@@ -634,20 +644,21 @@ class SpiderFootCli(cmd.Cmd):c = self.myparseline(line)titles = dict()if "-x" in c[0]:- titles = {"0": "ID",- "1": "Name",- "2": "Target",- "4": "Started",- "5": "Finished",- "6": "Status",- "7": "Total Elements"+ titles = {+ "0": "ID",+ "1": "Name",+ "2": "Target",+ "4": "Started",+ "5": "Finished",+ "6": "Status",+ "7": "Total Elements"}else:titles = {- "0": "ID",- "2": "Target",- "6": "Status",- "7": "Total Elements"+ "0": "ID",+ "2": "Target",+ "6": "Status",+ "7": "Total Elements"}self.send_output(d, line, titles=titles)@@ -655,18 +666,18 @@ class SpiderFootCli(cmd.Cmd):# Show the data from a scan.def do_data(self, line):"""data <sid> [-t type] [-x] [-u]- Get the scan data for scan ID <sid> and optionally the element- type [type] (e.g. EMAILADDR), [type]. Use -x for extended format.+ Get the scan data for scan ID <sid> and optionally the element+ type [type] (e.g. EMAILADDR), [type]. Use -x for extended format.Use -u for a unique set of results."""c = self.myparseline(line)if len(c[0]) < 1:self.edprint("Invalid syntax.")return- post = { "id": c[0][0] }+ post = {"id": c[0][0]}if "-t" in c[0]:- post["eventType"] = c[0][c[0].index("-t")+1]+ post["eventType"] = c[0][c[0].index("-t") + 1]else:post["eventType"] = "ALL"@@ -684,13 +695,13 @@ class SpiderFootCli(cmd.Cmd):returnif "-u" in c[0]:- titles={- "0": "Data"+ titles = {+ "0": "Data"}else:- titles={- "10": "Type",- "1": "Data"+ titles = {+ "10": "Type",+ "1": "Data"}if "-x" in c[0]:titles["0"] = "Last Seen"@@ -711,16 +722,16 @@ class SpiderFootCli(cmd.Cmd):returnif '-t' in c[0]:- export_format = c[0][c[0].index("-t")+1]+ export_format = c[0][c[0].index("-t") + 1]else:export_format = 'json'- if not export_format in ['json', 'csv']:+ if export_format not in ['json', 'csv']:print("Invalid export format: %s" % export_format)returnbase_url = self.ownopts['cli.server_baseurl']- post = { "ids": c[0][0] }+ post = {"ids": c[0][0]}if export_format == 'json':res = self.request(base_url + '/scanexportjsonmulti', post=post)@@ -749,9 +760,9 @@ class SpiderFootCli(cmd.Cmd):# Show logs.def do_logs(self, line):"""logs <sid> [-l count] [-w]- Show the most recent [count] logs for a given scan ID, <sid>.+ Show the most recent [count] logs for a given scan ID, <sid>.If no count is supplied, all logs are given.- If -w is supplied, logs will be streamed to the console until+ If -w is supplied, logs will be streamed to the console untilCtrl-C is entered."""c = self.myparseline(line)if len(c[0]) < 1:@@ -760,11 +771,11 @@ class SpiderFootCli(cmd.Cmd):sid = c[0][0]limit = Noneif "-l" in c[0]:- limit = c[0][c[0].index("-l")+1]+ limit = c[0][c[0].index("-l") + 1]if "-w" not in c[0]:d = self.request(self.ownopts['cli.server_baseurl'] + "/scanlog",- post={ 'id': sid, 'limit': limit })+ post={'id': sid, 'limit': limit})if not d:returnj = json.loads(d)@@ -772,12 +783,16 @@ class SpiderFootCli(cmd.Cmd):self.dprint("No results.")return- self.send_output(d, line, titles={"0": "Generated",- "1": "Type",- "2": "Source",- "3": "Message"- }- )+ self.send_output(+ d,+ line,+ titles={+ "0": "Generated",+ "1": "Type",+ "2": "Source",+ "3": "Message"+ }+ )returnelse:# Get the rowid of the latest log message@@ -796,12 +811,13 @@ class SpiderFootCli(cmd.Cmd):if not limit:limit = 10d = self.request(self.ownopts['cli.server_baseurl'] + "/scanlog",- post={'id': sid, 'reverse': '1', 'rowId': rowid-int(limit)})+ post={'id': sid, 'reverse': '1', 'rowId': rowid - int(limit)})if not d:return+j = json.loads(d)for r in j:- #self.send_output(str(r), line, total=False, raw=True)+ # self.send_output(str(r), line, total=False, raw=True)if r[2] == "ERROR":self.edprint(r[1] + ": " + r[3])else:@@ -827,13 +843,13 @@ class SpiderFootCli(cmd.Cmd):# Start a new scan.def do_start(self, line):"""start <target> (-m m1,... | -t t1,... | -u case) [-n name] [-w]- Start a scan against <target> using modules m1,... OR looking+ Start a scan against <target> using modules m1,... OR lookingfor types t1,...OR by use case ("all", "investigate", "passive" and "footprint").- Scan be be optionally named [name], without a name the target+ Scan be be optionally named [name], without a name the targetwill be used.- Use -w to watch the logs from the scan. Ctrl-C to abort the+ Use -w to watch the logs from the scan. Ctrl-C to abort thelogging (but will not abort the scan)."""mods = ""@@ -845,15 +861,15 @@ class SpiderFootCli(cmd.Cmd):returnif "-m" in c[0]:- mods = c[0][c[0].index("-m")+1]+ mods = c[0][c[0].index("-m") + 1]if "-t" in c[0]:# Scan by type- types = c[0][c[0].index("-t")+1]+ types = c[0][c[0].index("-t") + 1]if "-u" in c[0]:# Scan by use case- usecase = c[0][c[0].index("-u")+1]+ usecase = c[0][c[0].index("-u") + 1]if not mods and not types and not usecase:self.edprint("Invalid syntax.")@@ -861,16 +877,17 @@ class SpiderFootCli(cmd.Cmd):target = c[0][0]if "-n" in c[0]:- title = c[0][c[0].index("-n")+1]+ title = c[0][c[0].index("-n") + 1]else:title = target- post = { "scanname": title,- "scantarget": target,- "modulelist": mods,- "typelist": types,- "usecase": usecase,- "cli": "1"+ post = {+ "scanname": title,+ "scantarget": target,+ "modulelist": mods,+ "typelist": types,+ "usecase": usecase,+ "cli": "1"}d = self.request(self.ownopts['cli.server_baseurl'] + "/startscan",post=post)@@ -898,8 +915,7 @@ class SpiderFootCli(cmd.Cmd):self.edprint("Invalid syntax.")return- d = self.request(self.ownopts['cli.server_baseurl'] + \- "/stopscan?id=" + id + "")+ d = self.request(self.ownopts['cli.server_baseurl'] + "/stopscan?id=" + id)if not d:return@@ -918,7 +934,7 @@ class SpiderFootCli(cmd.Cmd):# Search for datadef do_find(self, line):"""find "<string|/regex/>" <[-s sid]|[-t type]> [-x]- Search for string/regex, limited to the scope of either a scan ID or+ Search for string/regex, limited to the scope of either a scan ID orevent type. -x for extended format."""c = self.myparseline(line)if len(c[0]) < 1:@@ -928,14 +944,16 @@ class SpiderFootCli(cmd.Cmd):val = c[0][0]sid = Noneetype = None+if "-t" in c[0]:- etype = c[0][c[0].index("-t")+1]+ etype = c[0][c[0].index("-t") + 1]if "-s" in c[0]:- sid = c[0][c[0].index("-s")+1]+ sid = c[0][c[0].index("-s") + 1]- titles={"0": "Last Seen",- "1": "Data",- "3": "Module"+ titles = {+ "0": "Last Seen",+ "1": "Data",+ "3": "Module"}if "-x" in c[0]:titles["2"] = "Source Data"@@ -958,7 +976,7 @@ class SpiderFootCli(cmd.Cmd):# Summary of a scandef do_summary(self, line):"""summary <sid> [-t]- Summarise the results for a scan ID, <sid>. -t to only show+ Summarise the results for a scan ID, <sid>. -t to only showthe element types."""c = self.myparseline(line)if len(c[0]) < 1:@@ -970,15 +988,17 @@ class SpiderFootCli(cmd.Cmd):if "-t" in c[0]:titles = {"0": "Element Type"}else:- titles={"0": "Element Type",- "1": "Element Description",- "3": "Total",- "4": "Unique"+ titles = {+ "0": "Element Type",+ "1": "Element Description",+ "3": "Total",+ "4": "Unique"}d = self.request(self.ownopts['cli.server_baseurl'] + "/scansummary?id=" + sid + "&by=type")if not d:return+j = json.loads(d)if not j:@@ -1069,7 +1089,7 @@ class SpiderFootCli(cmd.Cmd):returnj = list()serverconfig = dict()- token = "" # nosec+ token = "" # nosecif not d:self.edprint("Unable to obtain SpiderFoot server-side config.")else:@@ -1111,9 +1131,12 @@ class SpiderFootCli(cmd.Cmd):self.dprint(k + " = " + str(serverconfig[k]), plain=True)if len(output) > 0:- self.send_output(json.dumps(output), line,- { 'opt': "Option", 'val': "Value"},- total=False)+ self.send_output(+ json.dumps(output),+ line,+ {'opt': "Option", 'val': "Value"},+ total=False+ )returnif val:@@ -1184,7 +1207,7 @@ class SpiderFootCli(cmd.Cmd):"""shellRun a shell command locally."""self.dprint("Running shell command:" + str(line))- self.dprint(os.popen(line).read(), plain=True) # nosec+ self.dprint(os.popen(line).read(), plain=True) # nosecdef do_clear(self, line):"""clear@@ -1204,6 +1227,7 @@ class SpiderFootCli(cmd.Cmd):print("\n")return True+if __name__ == "__main__":p = argparse.ArgumentParser(description='SpiderFoot: Open Source Intelligence Automation.')p.add_argument("-d", "--debug", help="Enable debug output.", action='store_true')diff --git a/sfdb.py b/sfdb.pyindex b230966b..7e106459 100644--- a/sfdb.py+++ b/sfdb.py@@ -16,6 +16,7 @@ import timeimport threadingfrom sflib import SpiderFoot, SpiderFootEvent+class SpiderFootDb:"""SpiderFoot database@@ -291,7 +292,7 @@ class SpiderFootDb:# a custom function to do so..def __dbregex__(qry, data):try:- rx = re.compile(qry, re.IGNORECASE|re.DOTALL)+ rx = re.compile(qry, re.IGNORECASE | re.DOTALL)ret = rx.match(data)except BaseException as e:return False@@ -319,7 +320,7 @@ class SpiderFootDb:except BaseException as e:continueself.conn.commit()- #self.conn.close()+ # self.conn.close()## Back-end database operations@@ -397,7 +398,7 @@ class SpiderFootDb:raise ValueError(f"No valid search criteria provided; expected: {', '.join(valid_criteria)}")if len(criteria) == 1:- raise ValueError(f"Only one search criteria provided; expected at least two")+ raise ValueError("Only one search criteria provided; expected at least two")qvars = list()qry = "SELECT ROUND(c.generated) AS generated, c.data, \@@ -434,8 +435,6 @@ class SpiderFootDb:with self.dbhLock:try:- #print(qry)- #print(str(qvars))self.dbh.execute(qry, qvars)return self.dbh.fetchall()except sqlite3.Error as e:@@ -473,7 +472,7 @@ class SpiderFootDb:Raises:IOError: database I/O failed-+Todo:Do something smarter to handle database locks"""@@ -502,8 +501,8 @@ class SpiderFootDb:self.conn.commit()except sqlite3.Error as e:if "locked" in e.args[0] or "thread" in e.args[0]:- #print("[warning] Couldn't log due to SQLite limitations. You can probably ignore this.")- #self.sf.fatal(f"Unable to log event in DB due to lock: {e.args[0]}")+ # print("[warning] Couldn't log due to SQLite limitations. You can probably ignore this.")+ # self.sf.fatal(f"Unable to log event in DB due to lock: {e.args[0]}")passelse:raise IOError(f"Unable to log scan event in DB: {e.args[0]}")@@ -697,7 +696,7 @@ class SpiderFootDb:if not isinstance(eventType, str):raise TypeError(f"eventType is {type(eventType)}; expected str()")-+qry = "SELECT ROUND(c.generated) AS generated, c.data, \s.data as 'source_data', \c.module, c.type, c.confidence, c.visibility, c.risk, c.hash, \@@ -747,7 +746,7 @@ class SpiderFootDb:if not isinstance(eventType, str):raise TypeError(f"eventType is {type(eventType)}; expected str()")-+qry = "SELECT DISTINCT data, type, COUNT(*) FROM tbl_scan_results \WHERE scan_instance_id = ?"qvars = [instanceId]@@ -940,7 +939,7 @@ class SpiderFootDb:raise TypeError(f"optMap is {type(optMap)}; expected dict()")if not optMap:raise ValueError("optMap is empty")-+qry = "REPLACE INTO tbl_config (scope, opt, val) VALUES (?, ?, ?)"with self.dbhLock:@@ -974,7 +973,7 @@ class SpiderFootDb:"""qry = "SELECT scope, opt, val FROM tbl_config"-+retval = dict()with self.dbhLock:@@ -1029,7 +1028,7 @@ class SpiderFootDb:raise TypeError(f"optMap is {type(optMap)}; expected dict()")if not optMap:raise ValueError("optMap is empty")-+qry = "REPLACE INTO tbl_scan_config \(scan_instance_id, component, opt, val) VALUES (?, ?, ?, ?)"@@ -1458,7 +1457,7 @@ class SpiderFootDb:while keepGoing:nextSet = self.scanElementChildrenDirect(instanceId, nextIds)- if nextSet == None or len(nextSet) == 0:+ if nextSet is None or len(nextSet) == 0:keepGoing = Falsebreakdiff --git a/sflib.py b/sflib.pyindex d15e6cb7..a0fbd056 100644--- a/sflib.py+++ b/sflib.py@@ -16,6 +16,7 @@ from stem.control import Controllerimport inspectimport hashlibimport html+import ioimport urllib.request, urllib.parse, urllib.errorimport jsonimport re@@ -27,21 +28,18 @@ import sslimport sysimport timeimport netaddr-import urllib.request, urllib.error, urllib.parse-import threadingimport tracebackimport OpenSSLimport uuidimport cryptographyimport dns.resolver+import urllib3from publicsuffixlist import PublicSuffixListfrom networkx import nxfrom networkx.readwrite.gexf import GEXFWriterfrom datetime import datetimefrom bs4 import BeautifulSoup, SoupStrainerfrom copy import deepcopy-import io-import urllib3# For hiding the SSL warnings coming from the requests liburllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)@@ -56,7 +54,7 @@ class SpiderFoot:socksProxy (str): SOCKS proxyopts (dict): configuration options"""-+_dbh = None_scanId = None_socksProxy = None@@ -175,9 +173,9 @@ class SpiderFoot:f.close()return ret- except BaseException as b:+ except BaseException as e:if fatal:- self.error("Unable to open option file, " + fname + ".")+ self.error(f"Unable to open option file, {fname}.")else:return None@@ -245,15 +243,15 @@ class SpiderFoot:for [parent, id] in parents[entity]:if parent in entities:if entity != parent:- #print("Adding entity parent: " + parent)+ # print("Adding entity parent: " + parent)mapping.add((entity, parent))else:ppids = list()- #print("Checking " + parent + " for entityship.")+ # print("Checking " + parent + " for entityship.")next_parents = get_next_parent_entities(parent, ppids)for next_parent in next_parents:if entity != next_parent:- #print("Adding next entity parent: " + next_parent)+ # print("Adding next entity parent: " + next_parent)mapping.add((entity, next_parent))return mapping@@ -293,7 +291,7 @@ class SpiderFoot:if dst in root:col = ["255", "0", "0"]node = graph.add_node(dst)- graph.node[dst]['viz'] = {'color': { 'r': col[0], 'g': col[1], 'b': col[2] } }+ graph.node[dst]['viz'] = {'color': {'r': col[0], 'g': col[1], 'b': col[2]}}nodelist[dst] = ncounterif src not in nodelist:@@ -301,7 +299,7 @@ class SpiderFoot:if src in root:col = ["255", "0", "0"]graph.add_node(src)- graph.node[src]['viz'] = {'color': { 'r': col[0], 'g': col[1], 'b': col[2] } }+ graph.node[src]['viz'] = {'color': {'r': col[0], 'g': col[1], 'b': col[2]}}nodelist[src] = ncountergraph.add_edge(src, dst)@@ -336,36 +334,47 @@ class SpiderFoot:# Leave out this special caseif dst == "ROOT" or src == "ROOT":continue+if dst not in nodelist:ncounter = ncounter + 1+if dst in root:col = "#f00"- ret['nodes'].append({'id': str(ncounter),- 'label': str(dst),- 'x': random.SystemRandom().randint(1, 1000),- 'y': random.SystemRandom().randint(1, 1000),- 'size': "1",- 'color': col++ ret['nodes'].append({+ 'id': str(ncounter),+ 'label': str(dst),+ 'x': random.SystemRandom().randint(1, 1000),+ 'y': random.SystemRandom().randint(1, 1000),+ 'size': "1",+ 'color': col})+nodelist[dst] = ncounterif src not in nodelist:+ ncounter = ncounter + 1+if src in root:col = "#f00"- ncounter = ncounter + 1- ret['nodes'].append({'id': str(ncounter),- 'label': str(src),- 'x': random.SystemRandom().randint(1, 1000),- 'y': random.SystemRandom().randint(1, 1000),- 'size': "1",- 'color': col++ ret['nodes'].append({+ 'id': str(ncounter),+ 'label': str(src),+ 'x': random.SystemRandom().randint(1, 1000),+ 'y': random.SystemRandom().randint(1, 1000),+ 'size': "1",+ 'color': col})+nodelist[src] = ncounterecounter = ecounter + 1- ret['edges'].append({'id': str(ecounter),- 'source': str(nodelist[src]),- 'target': str(nodelist[dst])++ ret['edges'].append({+ 'id': str(ecounter),+ 'source': str(nodelist[src]),+ 'target': str(nodelist[dst])})return json.dumps(ret)@@ -810,7 +819,7 @@ class SpiderFoot:# Parse the target and set the targetTypefor rxpair in regexToType:rx = list(rxpair.keys())[0]- if re.match(rx, target, re.IGNORECASE|re.UNICODE):+ if re.match(rx, target, re.IGNORECASE | re.UNICODE):targetType = list(rxpair.values())[0]breakreturn targetType@@ -1147,7 +1156,7 @@ class SpiderFoot:ps = PublicSuffixList(tldList, only_icann=True, accept_unknown=False)sfx = ps.privatesuffix(hostname)return sfx == hostname-+def validIP(self, address):"""Check if the provided string is a valid IPv4 address.@@ -1264,8 +1273,8 @@ class SpiderFoot:"""chars = ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm',- 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z',- '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '-', '.']+ 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z',+ '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '-', '.']for c in cmd:if c.lower() not in chars:return False@@ -1290,14 +1299,14 @@ class SpiderFoot:wd = dict()- dicts = [ "english", "german", "french", "spanish" ]+ dicts = ["english", "german", "french", "spanish"]for d in dicts:try:with io.open(self.myPath() + "/dicts/ispell/" + d + ".dict", 'r', encoding='utf8', errors='ignore') as wdct:dlines = wdct.readlines()except BaseException as e:- self.debug("Could not read dictionary: " + str(e))+ self.debug(f"Could not read dictionary: {e}")continuefor w in dlines:@@ -1315,7 +1324,7 @@ class SpiderFoot:wd = dict()- dicts = [ "names" ]+ dicts = ["names"]for d in dicts:try:@@ -1332,7 +1341,6 @@ class SpiderFoot:return list(wd.keys())-def dataParentChildToTree(self, data):"""Converts a dictionary of k -> array to a nestedtree that can be digested by d3 for visualizations.@@ -1500,7 +1508,7 @@ class SpiderFoot:t = target.targetTypev = target.targetValue- if t in [ "IP_ADDRESS", "IPV6_ADDRESS" ]:+ if t in ["IP_ADDRESS", "IPV6_ADDRESS"]:r = self.resolveIP(v)if r:ret.extend(r)@@ -1625,7 +1633,7 @@ class SpiderFoot:emails.add(match)return list(emails)-+def parseCreditCards(self, data):"""Find all credit card numbers with the supplied content.@@ -1644,13 +1652,13 @@ class SpiderFoot:if not isinstance(data, str):return list()- creditCards = set()+ creditCards = set()- # Remove whitespace from data.- # Credit cards might contain spaces between them+ # Remove whitespace from data.+ # Credit cards might contain spaces between them# which will cause regex mismatchdata = data.replace(" ", "")-+# Extract all numbers with lengths ranging from 13 - 19 digitsmatches = re.findall(r"[0-9]{13,19}", data)@@ -1680,7 +1688,7 @@ class SpiderFoot:else:self.debug("Skipped invalid credit card number: " + match)return list(creditCards)-+def parseIBANNumbers(self, data):"""Find all International Bank Account Numbers (IBANs) within the supplied content.@@ -1703,30 +1711,30 @@ class SpiderFoot:# Dictionary of country codes and their respective IBAN lengthsibanCountryLengths = {- "AL" : 28, "AD" : 24, "AT" : 20, "AZ" : 28,- "ME" : 22, "BH" : 22, "BY" : 28, "BE" : 16,- "BA" : 20, "BR" : 29, "BG" : 22, "CR" : 22,- "HR" : 21, "CY" : 28, "CZ" : 24, "DK" : 18,- "DO" : 28, "EG" : 29, "SV" : 28, "FO" : 18,- "FI" : 18, "FR" : 27, "GE" : 22, "DE" : 22,- "GI" : 23, "GR" : 27, "GL" : 18, "GT" : 28,- "VA" : 22, "HU" : 28, "IS" : 26, "IQ" : 23,- "IE" : 22, "IL" : 23, "JO" : 30, "KZ" : 20,- "XK" : 20, "KW" : 30, "LV" : 21, "LB" : 28,- "LI" : 21, "LT" : 20, "LU" : 20, "MT" : 31,- "MR" : 27, "MU" : 30, "MD" : 24, "MC" : 27,- "DZ" : 24, "AO" : 25, "BJ" : 28, "VG" : 24,- "BF" : 27, "BI" : 16, "CM" : 27, "CV" : 25,- "CG" : 27, "EE" : 20, "GA" : 27, "GG" : 22,- "IR" : 26, "IM" : 22, "IT" : 27, "CI" : 28,- "JE" : 22, "MK" : 19, "MG" : 27, "ML" : 28,- "MZ" : 25, "NL" : 18, "NO" : 15, "PK" : 24,- "PS" : 29, "PL" : 28, "PT" : 25, "QA" : 29,- "RO" : 24, "LC" : 32, "SM" : 27, "ST" : 25,- "SA" : 24, "SN" : 28, "RS" : 22, "SC" : 31,- "SK" : 24, "SI" : 19, "ES" : 24, "CH" : 21,- "TL" : 23, "TN" : 24, "TR" : 26, "UA" : 29,- "AE" : 23, "GB" : 22, "SE" : 24+ "AL": 28, "AD": 24, "AT": 20, "AZ": 28,+ "ME": 22, "BH": 22, "BY": 28, "BE": 16,+ "BA": 20, "BR": 29, "BG": 22, "CR": 22,+ "HR": 21, "CY": 28, "CZ": 24, "DK": 18,+ "DO": 28, "EG": 29, "SV": 28, "FO": 18,+ "FI": 18, "FR": 27, "GE": 22, "DE": 22,+ "GI": 23, "GR": 27, "GL": 18, "GT": 28,+ "VA": 22, "HU": 28, "IS": 26, "IQ": 23,+ "IE": 22, "IL": 23, "JO": 30, "KZ": 20,+ "XK": 20, "KW": 30, "LV": 21, "LB": 28,+ "LI": 21, "LT": 20, "LU": 20, "MT": 31,+ "MR": 27, "MU": 30, "MD": 24, "MC": 27,+ "DZ": 24, "AO": 25, "BJ": 28, "VG": 24,+ "BF": 27, "BI": 16, "CM": 27, "CV": 25,+ "CG": 27, "EE": 20, "GA": 27, "GG": 22,+ "IR": 26, "IM": 22, "IT": 27, "CI": 28,+ "JE": 22, "MK": 19, "MG": 27, "ML": 28,+ "MZ": 25, "NL": 18, "NO": 15, "PK": 24,+ "PS": 29, "PL": 28, "PT": 25, "QA": 29,+ "RO": 24, "LC": 32, "SM": 27, "ST": 25,+ "SA": 24, "SN": 28, "RS": 22, "SC": 31,+ "SK": 24, "SI": 19, "ES": 24, "CH": 21,+ "TL": 23, "TN": 24, "TR": 26, "UA": 29,+ "AE": 23, "GB": 22, "SE": 24}# Normalize input data to remove whitespace@@ -1736,7 +1744,7 @@ class SpiderFoot:# and starting with two charactersmatches = re.findall("[A-Za-z]{2}[A-Za-z0-9]{13,30}", data)- for match in matches:+ for match in matches:iban = match.upper()countryCode = iban[0:2]@@ -1744,7 +1752,7 @@ class SpiderFoot:if countryCode not in ibanCountryLengths.keys():self.debug("Skipped invalid IBAN (invalid country code): %s" % iban)continue-+if len(iban) != ibanCountryLengths[countryCode]:self.debug("Skipped invalid IBAN (invalid length): %s" % iban)continue@@ -1830,7 +1838,7 @@ class SpiderFoot:if ret['expiry'] <= now:ret['expired'] = Trueexcept BaseException as e:- self.error("Error processing date in certificate: " + str(e) , False)+ self.error(f"Error processing date in certificate: {e}", False)ret['certerror'] = Truereturn ret@@ -1952,8 +1960,7 @@ class SpiderFoot:try:for t in list(tags.keys()):- for lnk in BeautifulSoup(data, "lxml",- parse_only=SoupStrainer(t)).find_all(t):+ for lnk in BeautifulSoup(data, "lxml", parse_only=SoupStrainer(t)).find_all(t):if lnk.has_attr(tags[t]):urlsRel.append(lnk[tags[t]])except BaseException as e:@@ -1973,10 +1980,10 @@ class SpiderFoot:# Don't include stuff likely part of some dynamically built incomplete# URL found in Javascript code (character is part of some logic)- if link[len(link) - 1] == '.' or link[0] == '+' or \- 'javascript:' in linkl or '()' in link:+ if link[len(link) - 1] == '.' or link[0] == '+' or 'javascript:' in linkl or '()' in link:self.debug('unlikely link: ' + link)continue+# Filter in-page linksif re.match('.*#.[^/]+', link):self.debug('in-page link: ' + link)@@ -2068,8 +2075,28 @@ class SpiderFoot:proxies = dict()if self.opts['_socks1type']:- neverProxyNames = [ self.opts['_socks2addr'] ]- neverProxySubnets = [ "192.168.", "127.", "10." ]+ neverProxyNames = [self.opts['_socks2addr']]+ neverProxySubnets = [+ "127."+ "10.",+ "192.168.",+ "172.16.",+ "172.17.",+ "172.18.",+ "172.19.",+ "172.20.",+ "172.21.",+ "172.22.",+ "172.23.",+ "172.24.",+ "172.25.",+ "172.26.",+ "172.27.",+ "172.28.",+ "172.29.",+ "172.30.",+ "172.31."+ ]host = self.urlFQDN(url)# Completely on or off?@@ -2118,12 +2145,15 @@ class SpiderFoot:if sizeLimit or headOnly:if not noLog:- self.info("Fetching (HEAD only): " + self.removeUrlCreds(url) + \- " [user-agent: " + header['User-Agent'] + "] [timeout: " + \- str(timeout) + "]")-- hdr = self.getSession().head(url, headers=header, proxies=proxies,- verify=verify, timeout=timeout)+ self.info(f"Fetching (HEAD only): {self.removeUrlCreds(url)} [user-agent: {header['User-Agent']}] [timeout: {timeout}]")++ hdr = self.getSession().head(+ url,+ headers=header,+ proxies=proxies,+ verify=verify,+ timeout=timeout+ )size = int(hdr.headers.get('content-length', 0))newloc = hdr.headers.get('location', url).strip()@@ -2141,12 +2171,15 @@ class SpiderFoot:if result['realurl'] != url:if not noLog:- self.info("Fetching (HEAD only): " + self.removeUrlCreds(url) + \- " [user-agent: " + header['User-Agent'] + "] [timeout: " + \- str(timeout) + "]")-- hdr = self.getSession().head(result['realurl'], headers=header, proxies=proxies,- verify=verify, timeout=timeout)+ self.info(f"Fetching (HEAD only): {self.removeUrlCreds(url)} [user-agent: {header['User-Agent']}] [timeout: {timeout}]")++ hdr = self.getSession().head(+ result['realurl'],+ headers=header,+ proxies=proxies,+ verify=verify,+ timeout=timeout+ )size = int(hdr.headers.get('content-length', 0))result['realurl'] = hdr.headers.get('location', result['realurl'])result['code'] = str(hdr.status_code)@@ -2156,21 +2189,32 @@ class SpiderFoot:if not noLog:if cookies is None:- self.info("Fetching: " + self.removeUrlCreds(url) + " [user-agent: " + \- header['User-Agent'] + "] [timeout: " + str(timeout) + "]")+ self.info(f"Fetching: {self.removeUrlCreds(url)} [user-agent: {header['User-Agent']}] [timeout: {timeout}]")else:- self.info("Fetching (incl. cookies): " + self.removeUrlCreds(url) + \- " [user-agent: " + header['User-Agent'] + "] [timeout: " + \- str(timeout) + "]")+ self.info(f"Fetching (incl. cookies): {self.removeUrlCreds(url)} [user-agent: {header['User-Agent']}] [timeout: {timeout}]")try:if postData:- res = self.getSession().post(url, data=postData, headers=header, proxies=proxies,- allow_redirects=True, cookies=cookies,- timeout=timeout, verify=verify)+ res = self.getSession().post(+ url,+ data=postData,+ headers=header,+ proxies=proxies,+ allow_redirects=True,+ cookies=cookies,+ timeout=timeout,+ verify=verify+ )else:- res = self.getSession().get(url, headers=header, proxies=proxies, allow_redirects=True,- cookies=cookies, timeout=timeout, verify=verify)+ res = self.getSession().get(+ url,+ headers=header,+ proxies=proxies,+ allow_redirects=True,+ cookies=cookies,+ timeout=timeout,+ verify=verify+ )except requests.exceptions.RequestException:raise Exception('Failed to connect to %s' % url) from None@@ -2234,7 +2278,7 @@ class SpiderFoot:m = mod.__name__atime = time.time()t = str(atime - btime)- self.info("Fetched data: " + str(len(result['content'] or '')) + \+ self.info("Fetched data: " + str(len(result['content'] or '')) +" (" + self.removeUrlCreds(url) + "), took " + t + "s")return result@@ -2421,7 +2465,7 @@ class SpiderFootPlugin(object):_priority = 1# Error state of the moduleerrorState = False-+def __init__(self):"""Not really needed in most cases."""pass@@ -2542,13 +2586,16 @@ class SpiderFootPlugin(object):if self.__outputFilter__:# Be strict about what events to pass on, unless they are# the ROOT event or the event type of the target.- if eventName != 'ROOT' and eventName != self.getTarget().targetType \- and eventName not in self.__outputFilter__:- return None+ if eventName != 'ROOT' and eventName != self.getTarget().targetType:+ if eventName not in self.__outputFilter__:+ return NonestoreOnly = False # Under some conditions, only store and don't notify- if sfEvent.data is None or (type(sfEvent.data) is str and len(sfEvent.data) == 0):+ if sfEvent.data is None:+ return None++ if type(sfEvent.data) is str and len(sfEvent.data) == 0:return Noneif self.checkForStop():@@ -2571,8 +2618,7 @@ class SpiderFootPlugin(object):prevEvent = sfEvent.sourceEventwhile prevEvent is not None:if prevEvent.sourceEvent is not None:- if prevEvent.sourceEvent.eventType == sfEvent.eventType and \- prevEvent.sourceEvent.data.lower() == sfEvent.data.lower():+ if prevEvent.sourceEvent.eventType == sfEvent.eventType and prevEvent.sourceEvent.data.lower() == sfEvent.data.lower():storeOnly = TruebreakprevEvent = prevEvent.sourceEvent@@ -2715,7 +2761,7 @@ class SpiderFootTarget(object):def targetType(self, targetType):if not isinstance(targetType, str):raise TypeError(f"targetType is {type(targetType)}; expected str()")-+if targetType not in self._validTypes:raise ValueError(f"targetType value is {targetType}; expected {self._validTypes}")@@ -2760,7 +2806,7 @@ class SpiderFootTarget(object):if value is None:return-+if {'type': typeName, 'value': value} in self.targetAliases:return None@@ -2851,7 +2897,7 @@ class SpiderFootTarget(object):# We can't really say anything about names, username or phone numbers,# so everything matches- if self.targetType in ["HUMAN_NAME", "PHONE_NUMBER", "USERNAME" ]:+ if self.targetType in ["HUMAN_NAME", "PHONE_NUMBER", "USERNAME"]:return Trueif netaddr.valid_ipv4(value):@@ -2862,9 +2908,8 @@ class SpiderFootTarget(object):if self.targetType == "NETBLOCK_OWNER":if netaddr.IPAddress(value) in netaddr.IPNetwork(self.targetValue):return True- if self.targetType in [ "IP_ADDRESS", "IPV6_ADDRESS" ]:- if netaddr.IPAddress(value) in \- netaddr.IPNetwork(netaddr.IPAddress(self.targetValue)):+ if self.targetType in ["IP_ADDRESS", "IPV6_ADDRESS"]:+ if netaddr.IPAddress(value) in netaddr.IPNetwork(netaddr.IPAddress(self.targetValue)):return Trueelse:for name in self.getNames():@@ -3025,7 +3070,7 @@ class SpiderFootEvent(object):TypeError: confidence type was invalidValueError: confidence value was invalid"""-+if not isinstance(eventType, str):raise TypeError(f"eventType is {type(eventType)}; expected str()")@@ -3044,7 +3089,7 @@ class SpiderFootEvent(object):TypeError: confidence type was invalidValueError: confidence value was invalid"""-+if not isinstance(confidence, int):raise TypeError(f"confidence is {type(confidence)}; expected int()")@@ -3063,7 +3108,7 @@ class SpiderFootEvent(object):TypeError: visibility type was invalidValueError: visibility value was invalid"""-+if not isinstance(visibility, int):raise TypeError(f"visibility is {type(visibility)}; expected int()")@@ -3082,7 +3127,7 @@ class SpiderFootEvent(object):TypeError: risk type was invalidValueError: risk value was invalid"""-+if not isinstance(risk, int):raise TypeError(f"risk is {type(risk)}; expected int()")@@ -3098,7 +3143,7 @@ class SpiderFootEvent(object):TypeError: module type was invalidValueError: module value was invalid"""-+if not isinstance(module, str):raise TypeError(f"module is {type(module )}; expected str()")@@ -3115,7 +3160,7 @@ class SpiderFootEvent(object):TypeError: data type was invalidValueError: data value was invalid"""-+if not isinstance(data, str):raise TypeError(f"data is {type(data)}; expected str()")@@ -3130,7 +3175,7 @@ class SpiderFootEvent(object):Raises:TypeError: sourceEvent type was invalid"""-+# "ROOT" is a special "hash" reserved for elements with no parent,# such as targets provided via the web UI or CLI.if self.eventType == "ROOT":diff --git a/sfscan.py b/sfscan.pyindex b8274ff0..f6778aaa 100644--- a/sfscan.py+++ b/sfscan.py@@ -14,11 +14,11 @@ import timeimport sysimport socketimport dns.resolver-import randomfrom copy import deepcopyfrom sfdb import SpiderFootDbfrom sflib import SpiderFoot, SpiderFootEvent, SpiderFootTarget, SpiderFootPlugin+class SpiderFootScanner():"""SpiderFootScanner object.@@ -337,9 +337,9 @@ class SpiderFootScanner():self.__setStatus("FINISHED", None, time.time() * 1000)except BaseException as e:exc_type, exc_value, exc_traceback = sys.exc_info()- self.__sf.error("Unhandled exception (" + e.__class__.__name__ + ") " + \- "encountered during scan. Please report this as a bug: " + \- repr(traceback.format_exception(exc_type, exc_value, exc_traceback)), False)+ self.__sf.error("Unhandled exception (" + e.__class__.__name__ + ") " ++ "encountered during scan. Please report this as a bug: " ++ repr(traceback.format_exception(exc_type, exc_value, exc_traceback)), False)self.__sf.status("Scan [" + self.__scanId + "] failed: " + str(e))self.__setStatus("ERROR-FAILED", None, time.time() * 1000)