SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.
Python attacksurfacecticybersecurityfootprintinghackinginformation-gatheringinformation-securityinfosecintelligence-gatheringosintosint-frameworkosint-reconnaissanceosint-toolpentestingpythonreconsecurity-toolsthreat-intelligencethreatintel
Flake8: Resolve various I Q0 SIM B9 A D violations (#1321)
bcoles18f71015e9ec
diff --git a/modules/sfp_alienvault.py b/modules/sfp_alienvault.pyindex 78b4e070..35ba0188 100644--- a/modules/sfp_alienvault.py+++ b/modules/sfp_alienvault.py@@ -15,9 +15,10 @@ import timeimport urllib.errorimport urllib.parseimport urllib.request-from datetime import datetime+from netaddr import IPNetwork+from spiderfoot import SpiderFootEvent, SpiderFootPlugindiff --git a/modules/sfp_alienvaultiprep.py b/modules/sfp_alienvaultiprep.pyindex aa14694d..23fcf517 100644--- a/modules/sfp_alienvaultiprep.py+++ b/modules/sfp_alienvaultiprep.py@@ -12,6 +12,7 @@# -------------------------------------------------------------------------------from netaddr import IPAddress, IPNetwork+from spiderfoot import SpiderFootEvent, SpiderFootPlugindiff --git a/modules/sfp_c99.py b/modules/sfp_c99.pyindex f9fe8152..6c31aa5a 100644--- a/modules/sfp_c99.py+++ b/modules/sfp_c99.py@@ -11,7 +11,8 @@# -------------------------------------------------------------------------------import json-from spiderfoot import SpiderFootPlugin, SpiderFootEvent++from spiderfoot import SpiderFootEvent, SpiderFootPluginclass sfp_c99(SpiderFootPlugin):diff --git a/modules/sfp_citadel.py b/modules/sfp_citadel.pyindex fcb1fb50..d0f1d507 100644--- a/modules/sfp_citadel.py+++ b/modules/sfp_citadel.py@@ -11,10 +11,10 @@# -------------------------------------------------------------------------------import json+import timeimport urllib.errorimport urllib.parseimport urllib.request-import timefrom spiderfoot import SpiderFootEvent, SpiderFootPlugindiff --git a/modules/sfp_debounce.py b/modules/sfp_debounce.pyindex 8f9ce70c..24eed127 100644--- a/modules/sfp_debounce.py+++ b/modules/sfp_debounce.py@@ -12,6 +12,7 @@# -------------------------------------------------------------------------------import json+from spiderfoot import SpiderFootEvent, SpiderFootPlugindiff --git a/modules/sfp_dehashed.py b/modules/sfp_dehashed.pyindex c750632f..91d52701 100644--- a/modules/sfp_dehashed.py+++ b/modules/sfp_dehashed.py@@ -9,9 +9,10 @@# Licence: GPL# -------------------------------------------------------------------------------+import base64import jsonimport time-import base64+from spiderfoot import SpiderFootEvent, SpiderFootPlugindiff --git a/modules/sfp_dnsdb.py b/modules/sfp_dnsdb.pyindex 521ccd83..b0cfb825 100644--- a/modules/sfp_dnsdb.py+++ b/modules/sfp_dnsdb.py@@ -12,9 +12,9 @@import jsonimport re+import timefrom spiderfoot import SpiderFootEvent, SpiderFootPlugin-import timeclass sfp_dnsdb(SpiderFootPlugin):diff --git a/modules/sfp_dnsdumpster.py b/modules/sfp_dnsdumpster.pyindex fda6f547..d6833188 100644--- a/modules/sfp_dnsdumpster.py+++ b/modules/sfp_dnsdumpster.py@@ -12,6 +12,7 @@# -------------------------------------------------------------------------------import re+from bs4 import BeautifulSoupfrom spiderfoot import SpiderFootEvent, SpiderFootPlugindiff --git a/modules/sfp_emailformat.py b/modules/sfp_emailformat.pyindex dfdc2132..cfd0d94a 100644--- a/modules/sfp_emailformat.py+++ b/modules/sfp_emailformat.py@@ -12,6 +12,7 @@# -------------------------------------------------------------------------------import re+from bs4 import BeautifulSoupfrom spiderfoot import SpiderFootEvent, SpiderFootPlugindiff --git a/modules/sfp_filemeta.py b/modules/sfp_filemeta.pyindex 84b14663..4e2e2a12 100644--- a/modules/sfp_filemeta.py+++ b/modules/sfp_filemeta.py@@ -14,10 +14,13 @@import ioimport mimetypes+import PyPDF2+import docx+import exifread+import pptx-import PyPDF2from spiderfoot import SpiderFootEvent, SpiderFootPlugindiff --git a/modules/sfp_gleif.py b/modules/sfp_gleif.pyindex 02f39de9..746d96fa 100644--- a/modules/sfp_gleif.py+++ b/modules/sfp_gleif.py@@ -11,10 +11,10 @@# Licence: GPL# --------------------------------------------------------------------------------import urllibimport json+import urllib-from spiderfoot import SpiderFootEvent, SpiderFootPlugin, SpiderFootHelpers+from spiderfoot import SpiderFootEvent, SpiderFootHelpers, SpiderFootPluginclass sfp_gleif(SpiderFootPlugin):diff --git a/modules/sfp_googlemaps.py b/modules/sfp_googlemaps.pyindex 71452f28..36c218df 100644--- a/modules/sfp_googlemaps.py+++ b/modules/sfp_googlemaps.py@@ -11,8 +11,8 @@# Licence: GPL# --------------------------------------------------------------------------------import urllibimport json+import urllibfrom spiderfoot import SpiderFootEvent, SpiderFootPlugindiff --git a/modules/sfp_grayhatwarfare.py b/modules/sfp_grayhatwarfare.pyindex 5817b4b7..575f355e 100644--- a/modules/sfp_grayhatwarfare.py+++ b/modules/sfp_grayhatwarfare.py@@ -12,6 +12,7 @@import jsonimport timeimport urllib+from spiderfoot import SpiderFootEvent, SpiderFootPlugindiff --git a/modules/sfp_haveibeenpwned.py b/modules/sfp_haveibeenpwned.pyindex ce97fb31..6b5d3dcf 100644--- a/modules/sfp_haveibeenpwned.py+++ b/modules/sfp_haveibeenpwned.py@@ -10,8 +10,8 @@# -------------------------------------------------------------------------------import json-import timeimport re+import timefrom spiderfoot import SpiderFootEvent, SpiderFootPlugin@@ -216,7 +216,7 @@ class sfp_haveibeenpwned(SpiderFootPlugin):try:source = n.get("Source")site = source- if source in sites.keys():+ if source in sites:site = f"{sites[n.get('Source')]}{n.get('Id')}"links.add(site)diff --git a/modules/sfp_multiproxy.py b/modules/sfp_multiproxy.pyindex f804697f..82343ea0 100644--- a/modules/sfp_multiproxy.py+++ b/modules/sfp_multiproxy.py@@ -12,6 +12,7 @@# -------------------------------------------------------------------------------from netaddr import IPAddress, IPNetwork+from spiderfoot import SpiderFootEvent, SpiderFootPlugindiff --git a/modules/sfp_nameapi.py b/modules/sfp_nameapi.pyindex 05b2e344..0fd58c43 100644--- a/modules/sfp_nameapi.py+++ b/modules/sfp_nameapi.py@@ -12,6 +12,7 @@# -------------------------------------------------------------------------------import json+from spiderfoot import SpiderFootEvent, SpiderFootPlugindiff --git a/modules/sfp_opencorporates.py b/modules/sfp_opencorporates.pyindex c77bea00..f0aa193a 100644--- a/modules/sfp_opencorporates.py+++ b/modules/sfp_opencorporates.py@@ -11,8 +11,8 @@# Licence: GPL# --------------------------------------------------------------------------------import urllibimport json+import urllibfrom spiderfoot import SpiderFootEvent, SpiderFootPlugindiff --git a/modules/sfp_portscan_tcp.py b/modules/sfp_portscan_tcp.pyindex bd0d591a..cd9bbaf2 100644--- a/modules/sfp_portscan_tcp.py+++ b/modules/sfp_portscan_tcp.py@@ -16,6 +16,7 @@ import threadingimport timefrom netaddr import IPNetwork+from spiderfoot import SpiderFootEvent, SpiderFootPlugindiff --git a/modules/sfp_spider.py b/modules/sfp_spider.pyindex d4799f05..8196101f 100644--- a/modules/sfp_spider.py+++ b/modules/sfp_spider.py@@ -14,7 +14,7 @@import jsonimport time-from spiderfoot import SpiderFootEvent, SpiderFootPlugin, SpiderFootHelpers+from spiderfoot import SpiderFootEvent, SpiderFootHelpers, SpiderFootPluginclass sfp_spider(SpiderFootPlugin):diff --git a/modules/sfp_torexits.py b/modules/sfp_torexits.pyindex 7c658ef8..b29debd0 100644--- a/modules/sfp_torexits.py+++ b/modules/sfp_torexits.py@@ -11,6 +11,7 @@# -------------------------------------------------------------------------------import re+from netaddr import IPAddress, IPNetworkfrom spiderfoot import SpiderFootEvent, SpiderFootPlugindiff --git a/modules/sfp_trashpanda.py b/modules/sfp_trashpanda.pyindex ed5656bc..50cee9f3 100644--- a/modules/sfp_trashpanda.py+++ b/modules/sfp_trashpanda.py@@ -11,8 +11,8 @@# Licence: GPL# --------------------------------------------------------------------------------import jsonimport base64+import jsonimport refrom spiderfoot import SpiderFootEvent, SpiderFootPlugindiff --git a/modules/sfp_trumail.py b/modules/sfp_trumail.pyindex 4d160766..676da5b1 100644--- a/modules/sfp_trumail.py+++ b/modules/sfp_trumail.py@@ -12,6 +12,7 @@# -------------------------------------------------------------------------------import json+from spiderfoot import SpiderFootEvent, SpiderFootPlugindiff --git a/modules/sfp_voipbl.py b/modules/sfp_voipbl.pyindex 609f2bbd..33a87d4b 100644--- a/modules/sfp_voipbl.py+++ b/modules/sfp_voipbl.py@@ -12,6 +12,7 @@# -------------------------------------------------------------------------------from netaddr import IPAddress, IPNetwork+from spiderfoot import SpiderFootEvent, SpiderFootPlugindiff --git a/modules/sfp_whois.py b/modules/sfp_whois.pyindex d41ab885..75051a15 100644--- a/modules/sfp_whois.py+++ b/modules/sfp_whois.py@@ -12,11 +12,13 @@# -------------------------------------------------------------------------------import ipwhois-import whois+from netaddr import IPAddressfrom spiderfoot import SpiderFootEvent, SpiderFootPlugin+import whois+class sfp_whois(SpiderFootPlugin):diff --git a/setup.cfg b/setup.cfgindex 918512b0..3e06c196 100644--- a/setup.cfg+++ b/setup.cfg@@ -1,27 +1,30 @@[flake8]flake8-max-line-length = 120-max-line-length = 120-max-complexity = 80+max-complexity = 60select = C,E,F,W,I,B,DUO,D,R,A,S,Q0,SIM,SFS-# Note: B9 and SIM tests should be reviewed and fixed instead of ignored-ignore = E501 W503 B006 E800 B9 SFS301 SF01 Q000 SIM102 SIM110 SIM111 SIM113 SIM115 SIM118+# Note: D and SIM tests should be reviewed and fixed instead of ignored+ignore = E501 W503 B006 E800 SFS301 SF01 Q000 SIM102 SIM113 SIM115 D100 D104# Note: most of these should be fixed instead of ignoredper-file-ignores =spiderfoot/event.py:A,D- spiderfoot/plugin.py:I,D- spiderfoot/db.py:A,D,SFS101,SIM106- sf.py:I,D,SFS101,SFS201- sflib.py:I,D,SFS101- sfwebui.py:A,I,D,SFS101,Q003- sfscan.py:I,D,SIM105,SIM109- modules/*:I,D+ spiderfoot/plugin.py:D,B902+ spiderfoot/db.py:D,SFS101,B902+ spiderfoot/helpers.py:D+ spiderfoot/target.py:D+ spiderfoot/logger.py:D+ sf.py:I,D,SFS201,B902+ sflib.py:I,D,SFS101,SIM110,SIM111,B902+ sfwebui.py:A,B902+ sfscan.py:I,D,SIM105,B902+ modules/*:D,B902+ modules/sfp_binaryedge.py:D,B902,C901modules/sfp__stor_stdout.py:D,SFS201modules/sfp_bitcoin.py:D,SFS101modules/sfp_reversewhois.py:D,R504- modules/sfp_zetalytics.py:D,R504- spiderfoot/*:I,D+ modules/sfp_ripe.py:D,SIM110,B902+ modules/sfp_zetalytics.py:D,R504,B902spiderfoot/__init__.py:I,D,F401- sfcli.py:D+ sfcli.py:D,B902test/*:I,D,SIM117docs/conf.py:A,Ddiff --git a/sfscan.py b/sfscan.pyindex d2e047d8..4f865e62 100644--- a/sfscan.py+++ b/sfscan.py@@ -165,7 +165,7 @@ class SpiderFootScanner():proxy_port = int(self.__config.get('_socks3port') or 0)if not proxy_port:- if proxy_type == '4' or proxy_type == '5':+ if proxy_type in ['4', '5']:proxy_port = 1080elif proxy_type.upper() == 'HTTP':proxy_port = 8080diff --git a/sfwebui.py b/sfwebui.pyindex 2b4846c8..5b372ec6 100644--- a/sfwebui.py+++ b/sfwebui.py@@ -15,7 +15,6 @@ import jsonimport loggingimport multiprocessing as mpimport random-import secureimport timefrom copy import deepcopyfrom io import StringIO@@ -23,14 +22,19 @@ from operator import itemgetterimport cherrypyfrom cherrypy import _cperror+from mako.lookup import TemplateLookupfrom mako.template import Template+import secure++from sflib import SpiderFoot++from sfscan import startSpiderFootScanner+from spiderfoot import SpiderFootDbfrom spiderfoot import SpiderFootHelpersfrom spiderfoot import __version__-from sflib import SpiderFoot-from sfscan import startSpiderFootScannerfrom spiderfoot.logger import logListenerSetup, logWorkerSetupmp.set_start_method("spawn", force=True)@@ -46,17 +50,17 @@ class SpiderFootWebUi:docroot = ''def __init__(self, web_config, config, loggingQueue=None):- """Initialize web server+ """Initialize web server.Args:- web_config: config settings for web interface (interface, port, root path)- config: SpiderFoot config+ web_config (dict): config settings for web interface (interface, port, root path)+ config (dict): SpiderFoot config+ loggingQueue: TBDRaises:TypeError: arg type is invalidValueError: arg value is invalid"""-if not isinstance(config, dict):raise TypeError(f"config is {type(config)}; expected dict()")if not config:@@ -115,8 +119,7 @@ class SpiderFootWebUi:})def error_page(self):- """Error page"""-+ """Error page."""cherrypy.response.status = 500if self.config['_debug']:@@ -125,7 +128,7 @@ class SpiderFootWebUi:cherrypy.response.body = b"<html><body>Error</body></html>"def error_page_401(self, status, message, traceback, version):- """Unauthorized access HTTP 401 error page+ """Unauthorized access HTTP 401 error page.Args:status (str): HTTP response status code and message@@ -139,7 +142,7 @@ class SpiderFootWebUi:return ""def error_page_404(self, status, message, traceback, version):- """Error page 404+ """Not found error page 404.Args:status (str): HTTP response status code and message@@ -154,7 +157,7 @@ class SpiderFootWebUi:return templ.render(message='Not Found', docroot=self.docroot, status=status, version=__version__)def jsonify_error(self, status, message):- """JSONify error response.+ """Jsonify error response.Args:status (str): HTTP response status code and message@@ -163,7 +166,6 @@ class SpiderFootWebUi:Returns:str: HTTP response template"""-cherrypy.response.headers['Content-Type'] = 'application/json'cherrypy.response.status = statusreturn {@@ -174,7 +176,7 @@ class SpiderFootWebUi:}def error(self, message):- """Generic error, but not exposed as not called directly+ """Show generic error page with error message.Args:message (str): error message@@ -182,7 +184,6 @@ class SpiderFootWebUi:Returns:None"""-templ = Template(filename='spiderfoot/templates/error.tmpl', lookup=self.lookup)return templ.render(message=message, docroot=self.docroot, version=__version__)@@ -198,7 +199,6 @@ class SpiderFootWebUi:Raises:TypeError: inputList type was invalid"""-if not isinstance(inputList, list):raise TypeError(f"inputList is {type(inputList)}; expected list()")@@ -209,7 +209,7 @@ class SpiderFootWebUi:ret.append('')continuec = html.escape(item, True)- c = c.replace('\'', '"')+ c = c.replace("'", '"')# We don't actually want & translated to &c = c.replace("&", "&").replace(""", "\"")ret.append(c)@@ -217,7 +217,7 @@ class SpiderFootWebUi:return retdef searchBase(self, id=None, eventType=None, value=None):- """Search+ """Search.Args:id: TBD@@ -274,7 +274,7 @@ class SpiderFootWebUi:@cherrypy.exposedef scaneventresultexport(self, id, type, dialect="excel"):- """Get scan event result data in CSV format+ """Get scan event result data in CSV format.Args:id (str): scan ID@@ -284,7 +284,6 @@ class SpiderFootWebUi:Returns:string: results in CSV format"""-dbh = SpiderFootDb(self.config)data = dbh.scanResultEvent(id, type)fileobj = StringIO()@@ -303,7 +302,7 @@ class SpiderFootWebUi:@cherrypy.exposedef scaneventresultexportmulti(self, ids, dialect="excel"):- """Get scan event result data in CSV format for multiple scans+ """Get scan event result data in CSV format for multiple scans.Args:ids (str): comma separated list of scan IDs@@ -312,7 +311,6 @@ class SpiderFootWebUi:Returns:string: results in CSV format"""-dbh = SpiderFootDb(self.config)scaninfo = dict()data = list()@@ -337,7 +335,7 @@ class SpiderFootWebUi:@cherrypy.exposedef scansearchresultexport(self, id, eventType=None, value=None, dialect="excel"):- """Get search result data in CSV format+ """Get search result data in CSV format.Args:id (str): scan ID@@ -348,7 +346,6 @@ class SpiderFootWebUi:Returns:string: results in CSV format"""-data = self.searchBase(id, eventType, value)fileobj = StringIO()parser = csv.writer(fileobj, dialect=dialect)@@ -367,7 +364,7 @@ class SpiderFootWebUi:@cherrypy.exposedef scanexportjsonmulti(self, ids):- """Get scan event result data in JSON format for multiple scans+ """Get scan event result data in JSON format for multiple scans.Args:ids (str): comma separated list of scan IDs@@ -375,7 +372,6 @@ class SpiderFootWebUi:Returns:string: results in CSV format"""-dbh = SpiderFootDb(self.config)scaninfo = list()scan_name = ""@@ -422,7 +418,7 @@ class SpiderFootWebUi:@cherrypy.exposedef scanviz(self, id, gexf="0"):- """Export entities from scan results for visualising+ """Export entities from scan results for visualising.Args:id (str): scan ID@@ -431,7 +427,6 @@ class SpiderFootWebUi:Returns:string: GEXF data"""-if not id:return None@@ -454,7 +449,7 @@ class SpiderFootWebUi:@cherrypy.exposedef scanvizmulti(self, ids, gexf="1"):- """Export entities results from multiple scans in GEXF format+ """Export entities results from multiple scans in GEXF format.Args:ids (str): scan IDs@@ -488,7 +483,7 @@ class SpiderFootWebUi:@cherrypy.expose@cherrypy.tools.json_out()def scanopts(self, id):- """Configuration used for a scan+ """Return configuration used for the specified scan as JSON.Args:id: scan ID@@ -535,7 +530,7 @@ class SpiderFootWebUi:@cherrypy.exposedef rerunscan(self, id):- """Rerun a scan+ """Rerun a scan.Args:id (str): scan ID@@ -546,7 +541,6 @@ class SpiderFootWebUi:Raises:HTTPRedirect: redirect to info page for new scan"""-# Snapshot the current configuration to be used by the scancfg = deepcopy(self.config)modlist = list()@@ -596,7 +590,7 @@ class SpiderFootWebUi:@cherrypy.exposedef rerunscanmulti(self, ids):- """Rerun scans+ """Rerun scans.Args:ids (str): comma separated list of scan IDs@@ -604,7 +598,6 @@ class SpiderFootWebUi:Returns:None"""-# Snapshot the current configuration to be used by the scancfg = deepcopy(self.config)modlist = list()@@ -652,12 +645,11 @@ class SpiderFootWebUi:@cherrypy.exposedef newscan(self):- """Configure a new scan+ """Configure a new scan.Returns:None"""-dbh = SpiderFootDb(self.config)types = dbh.eventTypes()templ = Template(filename='spiderfoot/templates/newscan.tmpl', lookup=self.lookup)@@ -668,7 +660,7 @@ class SpiderFootWebUi:@cherrypy.exposedef clonescan(self, id):"""- Clone an existing scan (pre-selected options in the newscan page)+ Clone an existing scan (pre-selected options in the newscan page).Args:id (str): scan ID to clone@@ -676,7 +668,6 @@ class SpiderFootWebUi:Returns:None"""-dbh = SpiderFootDb(self.config)types = dbh.eventTypes()info = dbh.scanInstanceGet(id)@@ -707,18 +698,17 @@ class SpiderFootWebUi:@cherrypy.exposedef index(self):- """Main page listing scans available+ """Show scan list page.Returns:None"""-templ = Template(filename='spiderfoot/templates/scanlist.tmpl', lookup=self.lookup)return templ.render(pageid='SCANLIST', docroot=self.docroot, version=__version__)@cherrypy.exposedef scaninfo(self, id):- """Information about a selected scan+ """Information about a selected scan.Args:id (str): scan id@@ -726,7 +716,6 @@ class SpiderFootWebUi:Returns:None"""-dbh = SpiderFootDb(self.config)res = dbh.scanInstanceGet(id)if res is None:@@ -738,7 +727,7 @@ class SpiderFootWebUi:@cherrypy.exposedef opts(self, updated=None):- """Settings+ """Show module and global settings page.Args:updated: TBD@@ -746,7 +735,6 @@ class SpiderFootWebUi:Returns:None"""-templ = Template(filename='spiderfoot/templates/opts.tmpl', lookup=self.lookup)self.token = random.SystemRandom().randint(0, 99999999)return templ.render(opts=self.config, pageid='SETTINGS', token=self.token, version=__version__,@@ -754,7 +742,7 @@ class SpiderFootWebUi:@cherrypy.exposedef optsexport(self, pattern=None):- """Export configuration+ """Export configuration.Args:pattern: TBD@@ -762,7 +750,6 @@ class SpiderFootWebUi:Returns:None"""-sf = SpiderFoot(self.config)conf = sf.configSerialize(self.config)content = ""@@ -773,9 +760,9 @@ class SpiderFootWebUi:if pattern:if pattern in opt:- content += "%s=%s\n" % (opt, conf[opt])+ content += f"{opt}={conf[opt]}\n"else:- content += "%s=%s\n" % (opt, conf[opt])+ content += f"{opt}={conf[opt]}\n"cherrypy.response.headers['Content-Disposition'] = 'attachment; filename="SpiderFoot.cfg"'cherrypy.response.headers['Content-Type'] = "text/plain"@@ -784,7 +771,7 @@ class SpiderFootWebUi:@cherrypy.expose@cherrypy.tools.json_out()def optsraw(self):- """Settings+ """Return global and module settings as json.Returns:str: settings as JSON@@ -808,7 +795,7 @@ class SpiderFootWebUi:@cherrypy.expose@cherrypy.tools.json_out()def scandelete(self, id):- """Delete scan(s)+ """Delete scan(s).Args:id (str): comma separated list of scan IDs@@ -837,7 +824,7 @@ class SpiderFootWebUi:@cherrypy.exposedef savesettings(self, allopts, token, configFile=None):- """Save settings, also used to completely reset them to default+ """Save settings, also used to completely reset them to default.Args:allopts: TBD@@ -850,9 +837,8 @@ class SpiderFootWebUi:Raises:HTTPRedirect: redirect to scan settings"""-if str(token) != str(self.token):- return self.error("Invalid token (%s)" % self.token)+ return self.error(f"Invalid token ({self.token})")if configFile: # configFile seems to get set even if a file isn't uploadedif configFile.file:@@ -875,7 +861,7 @@ class SpiderFootWebUi:allopts = json.dumps(tmp).encode('utf-8')except Exception as e:- return self.error("Failed to parse input file. Was it generated from SpiderFoot? (%s)" % e)+ return self.error(f"Failed to parse input file. Was it generated from SpiderFoot? ({e})")# Reset config to defaultif allopts == "RESET":@@ -899,13 +885,13 @@ class SpiderFootWebUi:self.config = sf.configUnserialize(cleanopts, currentopts)dbh.configSet(sf.configSerialize(self.config))except Exception as e:- return self.error("Processing one or more of your inputs failed: %s" % e)+ return self.error(f"Processing one or more of your inputs failed: {e}")raise cherrypy.HTTPRedirect(f"{self.docroot}/opts?updated=1")@cherrypy.exposedef savesettingsraw(self, allopts, token):- """Save settings, also used to completely reset them to default+ """Save settings, also used to completely reset them to default.Args:allopts: TBD@@ -914,11 +900,10 @@ class SpiderFootWebUi:Returns:str: save success as JSON"""-cherrypy.response.headers['Content-Type'] = "application/json; charset=utf-8"if str(token) != str(self.token):- return json.dumps(["ERROR", "Invalid token (%s)." % self.token]).encode('utf-8')+ return json.dumps(["ERROR", f"Invalid token ({self.token})."]).encode('utf-8')# Reset config to defaultif allopts == "RESET":@@ -942,7 +927,7 @@ class SpiderFootWebUi:self.config = sf.configUnserialize(cleanopts, currentopts)dbh.configSet(sf.configSerialize(self.config))except Exception as e:- return json.dumps(["ERROR", "Processing one or more of your inputs failed: %s" % e]).encode('utf-8')+ return json.dumps(["ERROR", f"Processing one or more of your inputs failed: {e}"]).encode('utf-8')return json.dumps(["SUCCESS", ""]).encode('utf-8')@@ -952,7 +937,6 @@ class SpiderFootWebUi:Returns:bool: success"""-try:dbh = SpiderFootDb(self.config)dbh.configClear() # Clear it in the DB@@ -964,7 +948,7 @@ class SpiderFootWebUi:@cherrypy.exposedef resultsetfp(self, id, resultids, fp):- """Set a bunch of results (hashes) as false positive+ """Set a bunch of results (hashes) as false positive.Args:id (str): scan ID@@ -974,7 +958,6 @@ class SpiderFootWebUi:Returns:str: set false positive status as JSON"""-cherrypy.response.headers['Content-Type'] = "application/json; charset=utf-8"dbh = SpiderFootDb(self.config)@@ -990,7 +973,7 @@ class SpiderFootWebUi:# Cannot set FPs if a scan is not completedstatus = dbh.scanInstanceGet(id)if not status:- return self.error("Invalid scan ID: %s" % id)+ return self.error(f"Invalid scan ID: {id}")if status[5] not in ["ABORTED", "FINISHED", "ERROR-FAILED"]:return json.dumps([@@ -1027,7 +1010,6 @@ class SpiderFootWebUi:Returns:str: list of event types"""-cherrypy.response.headers['Content-Type'] = "application/json; charset=utf-8"dbh = SpiderFootDb(self.config)@@ -1047,7 +1029,6 @@ class SpiderFootWebUi:Returns:str: list of modules"""-cherrypy.response.headers['Content-Type'] = "application/json; charset=utf-8"modinfo = list(self.config['__modules__'].keys())@@ -1099,7 +1080,7 @@ class SpiderFootWebUi:@cherrypy.exposedef startscan(self, scanname, scantarget, modulelist, typelist, usecase):- """Initiate a scan+ """Initiate a scan.Args:scanname (str): scan name@@ -1232,7 +1213,7 @@ class SpiderFootWebUi:@cherrypy.expose@cherrypy.tools.json_out()def stopscan(self, id):- """Stop a scan+ """Stop a scan.Args:id (str): comma separated list of scan IDs@@ -1274,10 +1255,10 @@ class SpiderFootWebUi:@cherrypy.expose@cherrypy.tools.json_out()def scanlog(self, id, limit=None, rowId=None, reverse=None):- """Scan log data+ """Scan log data.Args:- id: TBD+ id (str): scan IDlimit: TBDrowId: TBDreverse: TBD@@ -1357,7 +1338,7 @@ class SpiderFootWebUi:@cherrypy.expose@cherrypy.tools.json_out()def scanstatus(self, id):- """Basic information about a scan.+ """Show basic information about a scan, including status and number of each event type.Args:id (str): scan ID@@ -1414,7 +1395,7 @@ class SpiderFootWebUi:@cherrypy.expose@cherrypy.tools.json_out()def scaneventresults(self, id, eventType, filterfp=False):- """Event results for a scan.+ """Return all event results for a scan as JSON.Args:id (str): scan ID@@ -1454,7 +1435,7 @@ class SpiderFootWebUi:@cherrypy.expose@cherrypy.tools.json_out()def scaneventresultsunique(self, id, eventType, filterfp=False):- """Unique event results for a scan.+ """Return unique event results for a scan as JSON.Args:id (str): filter search results by scan IDdiff --git a/spiderfoot/db.py b/spiderfoot/db.pyindex 917af80a..132f12a0 100644--- a/spiderfoot/db.py+++ b/spiderfoot/db.py@@ -507,12 +507,11 @@ class SpiderFootDb:))self.conn.commit()except sqlite3.Error as e:- if "locked" in e.args[0] or "thread" in e.args[0]:- # print("[warning] Couldn't log due to SQLite limitations. You can probably ignore this.")- # log.critical(f"Unable to log event in DB due to lock: {e.args[0]}")- pass- else:+ if "locked" not in e.args[0] and "thread" not in e.args[0]:raise IOError(f"Unable to log scan event in DB: {e.args[0]}")+ # print("[warning] Couldn't log due to SQLite limitations. You can probably ignore this.")+ # log.critical(f"Unable to log event in DB due to lock: {e.args[0]}")+ passdef scanInstanceCreate(self, instanceId, scanName, scanTarget):"""Store a scan instance in the database.@@ -1000,7 +999,7 @@ class SpiderFootDb:except sqlite3.Error as e:raise IOError(f"Unable to clear configuration from the database: {e.args[0]}")- def scanConfigSet(self, id, optMap=dict()):+ def scanConfigSet(self, scan_id, optMap=dict()):"""Store a configuration value for a scan.Args:@@ -1026,10 +1025,10 @@ class SpiderFootDb:# Module optionif ":" in opt:parts = opt.split(':')- qvals = [id, parts[0], parts[1], optMap[opt]]+ qvals = [scan_id, parts[0], parts[1], optMap[opt]]else:# Global option- qvals = [id, "GLOBAL", opt, optMap[opt]]+ qvals = [scan_id, "GLOBAL", opt, optMap[opt]]try:self.dbh.execute(qry, qvals)diff --git a/spiderfoot/helpers.py b/spiderfoot/helpers.pyindex 3e787dba..3aace936 100644--- a/spiderfoot/helpers.py+++ b/spiderfoot/helpers.py@@ -2,6 +2,7 @@ import jsonimport randomimport reimport uuid+import networkx as nxfrom networkx.readwrite.gexf import GEXFWriter@@ -46,7 +47,7 @@ class SpiderFootHelpers():@staticmethoddef buildGraphGexf(root, title, data, flt=[]):- """Convert supplied raw data into GEXF (Graph Exchange XML Format) format (e.g. for Gephi)+ """Convert supplied raw data into GEXF (Graph Exchange XML Format) format (e.g. for Gephi).Args:root (str): TBD@@ -57,7 +58,6 @@ class SpiderFootHelpers():Returns:str: GEXF formatted XML"""-mapping = SpiderFootHelpers.buildGraphData(data, flt)graph = nx.Graph()@@ -109,7 +109,6 @@ class SpiderFootHelpers():Returns:str: TBD"""-mapping = SpiderFootHelpers.buildGraphData(data, flt)ret = dict()ret['nodes'] = list()@@ -247,7 +246,6 @@ class SpiderFootHelpers():Returns:dict: nested tree"""-if not isinstance(data, dict):# TODO: print errorreturn {}@@ -317,7 +315,6 @@ class SpiderFootHelpers():Returns:str: scan instance unique ID"""-return str(uuid.uuid4()).split("-")[0].upper()@staticmethod@@ -335,7 +332,6 @@ class SpiderFootHelpers():Fix whitespace parsing; ie, " " is not a valid disallowed path"""-returnArr = list()if not isinstance(robotsTxtData, str):diff --git a/spiderfoot/logger.py b/spiderfoot/logger.pyindex c7a554ae..dd08f717 100644--- a/spiderfoot/logger.py+++ b/spiderfoot/logger.py@@ -1,15 +1,16 @@-import sysimport logging-from pathlib import Path+import sysfrom contextlib import suppressfrom logging.handlers import QueueHandler, QueueListener+from pathlib import Path+from spiderfoot import SpiderFootDblog_format = logging.Formatter("%(asctime)s [%(levelname)s] %(message)s")class SpiderFootSqliteLogHandler(logging.Handler):- """Handler for logging to SQLite database+ """Handler for logging to SQLite database.This ensure all sqlite logging is done from a singleprocess and a single database handle.@@ -40,6 +41,7 @@ class SpiderFootSqliteLogHandler(logging.Handler):def logListenerSetup(loggingQueue, opts=None):"""Create and start a SpiderFoot log listener in its own thread.+This function should be called as soon as possible in the mainprocess, or whichever process is attached to stdin/stdout.diff --git a/spiderfoot/plugin.py b/spiderfoot/plugin.pyindex b32f4521..673d7525 100644--- a/spiderfoot/plugin.py+++ b/spiderfoot/plugin.py@@ -1,6 +1,6 @@import logging-import threadingimport queue+import threadingfrom time import sleepdiff --git a/spiderfoot/target.py b/spiderfoot/target.pyindex 5494df63..238aef2a 100644--- a/spiderfoot/target.py+++ b/spiderfoot/target.py@@ -25,7 +25,6 @@ class SpiderFootTarget():targetValue (str): target valuetypeName (str): target type"""-self.targetType = typeNameself.targetValue = targetValueself.targetAliases = list()@@ -102,7 +101,6 @@ class SpiderFootTarget():Returns:list: target aliases"""-ret = list()for item in self.targetAliases:if item['type'] == typeName:@@ -115,7 +113,6 @@ class SpiderFootTarget():Returns:list: domains associated with the target"""-e = self._getEquivalents("INTERNET_NAME")if self.targetType in ["INTERNET_NAME", "EMAILADDR"] and self.targetValue.lower() not in e:e.append(self.targetValue.lower())@@ -135,7 +132,6 @@ class SpiderFootTarget():Returns:list: TBD"""-e = self._getEquivalents("IP_ADDRESS")if self.targetType == "IP_ADDRESS":e.append(self.targetValue)@@ -147,8 +143,7 @@ class SpiderFootTarget():return edef matches(self, value, includeParents=False, includeChildren=True):- """Check whether the supplied value is "tightly" related- to the original target.+ """Check whether the supplied value is "tightly" related to the original target.Tightly in this case means: