Skip to content

SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.

Flake8: Resolve various I Q0 SIM B9 A D violations (#1321)

bcoles18f71015e9ec
diff --git a/modules/sfp_alienvault.py b/modules/sfp_alienvault.py
index 78b4e070..35ba0188 100644
--- a/modules/sfp_alienvault.py
+++ b/modules/sfp_alienvault.py
@@ -15,9 +15,10 @@ import time
import urllib.error
import urllib.parse
import urllib.request
-
from datetime import datetime
+
from netaddr import IPNetwork
+
from spiderfoot import SpiderFootEvent, SpiderFootPlugin
diff --git a/modules/sfp_alienvaultiprep.py b/modules/sfp_alienvaultiprep.py
index aa14694d..23fcf517 100644
--- a/modules/sfp_alienvaultiprep.py
+++ b/modules/sfp_alienvaultiprep.py
@@ -12,6 +12,7 @@
# -------------------------------------------------------------------------------
from netaddr import IPAddress, IPNetwork
+
from spiderfoot import SpiderFootEvent, SpiderFootPlugin
diff --git a/modules/sfp_c99.py b/modules/sfp_c99.py
index f9fe8152..6c31aa5a 100644
--- a/modules/sfp_c99.py
+++ b/modules/sfp_c99.py
@@ -11,7 +11,8 @@
# -------------------------------------------------------------------------------
import json
-from spiderfoot import SpiderFootPlugin, SpiderFootEvent
+
+from spiderfoot import SpiderFootEvent, SpiderFootPlugin
class sfp_c99(SpiderFootPlugin):
diff --git a/modules/sfp_citadel.py b/modules/sfp_citadel.py
index fcb1fb50..d0f1d507 100644
--- a/modules/sfp_citadel.py
+++ b/modules/sfp_citadel.py
@@ -11,10 +11,10 @@
# -------------------------------------------------------------------------------
import json
+import time
import urllib.error
import urllib.parse
import urllib.request
-import time
from spiderfoot import SpiderFootEvent, SpiderFootPlugin
diff --git a/modules/sfp_debounce.py b/modules/sfp_debounce.py
index 8f9ce70c..24eed127 100644
--- a/modules/sfp_debounce.py
+++ b/modules/sfp_debounce.py
@@ -12,6 +12,7 @@
# -------------------------------------------------------------------------------
import json
+
from spiderfoot import SpiderFootEvent, SpiderFootPlugin
diff --git a/modules/sfp_dehashed.py b/modules/sfp_dehashed.py
index c750632f..91d52701 100644
--- a/modules/sfp_dehashed.py
+++ b/modules/sfp_dehashed.py
@@ -9,9 +9,10 @@
# Licence: GPL
# -------------------------------------------------------------------------------
+import base64
import json
import time
-import base64
+
from spiderfoot import SpiderFootEvent, SpiderFootPlugin
diff --git a/modules/sfp_dnsdb.py b/modules/sfp_dnsdb.py
index 521ccd83..b0cfb825 100644
--- a/modules/sfp_dnsdb.py
+++ b/modules/sfp_dnsdb.py
@@ -12,9 +12,9 @@
import json
import re
+import time
from spiderfoot import SpiderFootEvent, SpiderFootPlugin
-import time
class sfp_dnsdb(SpiderFootPlugin):
diff --git a/modules/sfp_dnsdumpster.py b/modules/sfp_dnsdumpster.py
index fda6f547..d6833188 100644
--- a/modules/sfp_dnsdumpster.py
+++ b/modules/sfp_dnsdumpster.py
@@ -12,6 +12,7 @@
# -------------------------------------------------------------------------------
import re
+
from bs4 import BeautifulSoup
from spiderfoot import SpiderFootEvent, SpiderFootPlugin
diff --git a/modules/sfp_emailformat.py b/modules/sfp_emailformat.py
index dfdc2132..cfd0d94a 100644
--- a/modules/sfp_emailformat.py
+++ b/modules/sfp_emailformat.py
@@ -12,6 +12,7 @@
# -------------------------------------------------------------------------------
import re
+
from bs4 import BeautifulSoup
from spiderfoot import SpiderFootEvent, SpiderFootPlugin
diff --git a/modules/sfp_filemeta.py b/modules/sfp_filemeta.py
index 84b14663..4e2e2a12 100644
--- a/modules/sfp_filemeta.py
+++ b/modules/sfp_filemeta.py
@@ -14,10 +14,13 @@
import io
import mimetypes
+import PyPDF2
+
import docx
+
import exifread
+
import pptx
-import PyPDF2
from spiderfoot import SpiderFootEvent, SpiderFootPlugin
diff --git a/modules/sfp_gleif.py b/modules/sfp_gleif.py
index 02f39de9..746d96fa 100644
--- a/modules/sfp_gleif.py
+++ b/modules/sfp_gleif.py
@@ -11,10 +11,10 @@
# Licence: GPL
# -------------------------------------------------------------------------------
-import urllib
import json
+import urllib
-from spiderfoot import SpiderFootEvent, SpiderFootPlugin, SpiderFootHelpers
+from spiderfoot import SpiderFootEvent, SpiderFootHelpers, SpiderFootPlugin
class sfp_gleif(SpiderFootPlugin):
diff --git a/modules/sfp_googlemaps.py b/modules/sfp_googlemaps.py
index 71452f28..36c218df 100644
--- a/modules/sfp_googlemaps.py
+++ b/modules/sfp_googlemaps.py
@@ -11,8 +11,8 @@
# Licence: GPL
# -------------------------------------------------------------------------------
-import urllib
import json
+import urllib
from spiderfoot import SpiderFootEvent, SpiderFootPlugin
diff --git a/modules/sfp_grayhatwarfare.py b/modules/sfp_grayhatwarfare.py
index 5817b4b7..575f355e 100644
--- a/modules/sfp_grayhatwarfare.py
+++ b/modules/sfp_grayhatwarfare.py
@@ -12,6 +12,7 @@
import json
import time
import urllib
+
from spiderfoot import SpiderFootEvent, SpiderFootPlugin
diff --git a/modules/sfp_haveibeenpwned.py b/modules/sfp_haveibeenpwned.py
index ce97fb31..6b5d3dcf 100644
--- a/modules/sfp_haveibeenpwned.py
+++ b/modules/sfp_haveibeenpwned.py
@@ -10,8 +10,8 @@
# -------------------------------------------------------------------------------
import json
-import time
import re
+import time
from spiderfoot import SpiderFootEvent, SpiderFootPlugin
@@ -216,7 +216,7 @@ class sfp_haveibeenpwned(SpiderFootPlugin):
try:
source = n.get("Source")
site = source
- if source in sites.keys():
+ if source in sites:
site = f"{sites[n.get('Source')]}{n.get('Id')}"
links.add(site)
diff --git a/modules/sfp_multiproxy.py b/modules/sfp_multiproxy.py
index f804697f..82343ea0 100644
--- a/modules/sfp_multiproxy.py
+++ b/modules/sfp_multiproxy.py
@@ -12,6 +12,7 @@
# -------------------------------------------------------------------------------
from netaddr import IPAddress, IPNetwork
+
from spiderfoot import SpiderFootEvent, SpiderFootPlugin
diff --git a/modules/sfp_nameapi.py b/modules/sfp_nameapi.py
index 05b2e344..0fd58c43 100644
--- a/modules/sfp_nameapi.py
+++ b/modules/sfp_nameapi.py
@@ -12,6 +12,7 @@
# -------------------------------------------------------------------------------
import json
+
from spiderfoot import SpiderFootEvent, SpiderFootPlugin
diff --git a/modules/sfp_opencorporates.py b/modules/sfp_opencorporates.py
index c77bea00..f0aa193a 100644
--- a/modules/sfp_opencorporates.py
+++ b/modules/sfp_opencorporates.py
@@ -11,8 +11,8 @@
# Licence: GPL
# -------------------------------------------------------------------------------
-import urllib
import json
+import urllib
from spiderfoot import SpiderFootEvent, SpiderFootPlugin
diff --git a/modules/sfp_portscan_tcp.py b/modules/sfp_portscan_tcp.py
index bd0d591a..cd9bbaf2 100644
--- a/modules/sfp_portscan_tcp.py
+++ b/modules/sfp_portscan_tcp.py
@@ -16,6 +16,7 @@ import threading
import time
from netaddr import IPNetwork
+
from spiderfoot import SpiderFootEvent, SpiderFootPlugin
diff --git a/modules/sfp_spider.py b/modules/sfp_spider.py
index d4799f05..8196101f 100644
--- a/modules/sfp_spider.py
+++ b/modules/sfp_spider.py
@@ -14,7 +14,7 @@
import json
import time
-from spiderfoot import SpiderFootEvent, SpiderFootPlugin, SpiderFootHelpers
+from spiderfoot import SpiderFootEvent, SpiderFootHelpers, SpiderFootPlugin
class sfp_spider(SpiderFootPlugin):
diff --git a/modules/sfp_torexits.py b/modules/sfp_torexits.py
index 7c658ef8..b29debd0 100644
--- a/modules/sfp_torexits.py
+++ b/modules/sfp_torexits.py
@@ -11,6 +11,7 @@
# -------------------------------------------------------------------------------
import re
+
from netaddr import IPAddress, IPNetwork
from spiderfoot import SpiderFootEvent, SpiderFootPlugin
diff --git a/modules/sfp_trashpanda.py b/modules/sfp_trashpanda.py
index ed5656bc..50cee9f3 100644
--- a/modules/sfp_trashpanda.py
+++ b/modules/sfp_trashpanda.py
@@ -11,8 +11,8 @@
# Licence: GPL
# -------------------------------------------------------------------------------
-import json
import base64
+import json
import re
from spiderfoot import SpiderFootEvent, SpiderFootPlugin
diff --git a/modules/sfp_trumail.py b/modules/sfp_trumail.py
index 4d160766..676da5b1 100644
--- a/modules/sfp_trumail.py
+++ b/modules/sfp_trumail.py
@@ -12,6 +12,7 @@
# -------------------------------------------------------------------------------
import json
+
from spiderfoot import SpiderFootEvent, SpiderFootPlugin
diff --git a/modules/sfp_voipbl.py b/modules/sfp_voipbl.py
index 609f2bbd..33a87d4b 100644
--- a/modules/sfp_voipbl.py
+++ b/modules/sfp_voipbl.py
@@ -12,6 +12,7 @@
# -------------------------------------------------------------------------------
from netaddr import IPAddress, IPNetwork
+
from spiderfoot import SpiderFootEvent, SpiderFootPlugin
diff --git a/modules/sfp_whois.py b/modules/sfp_whois.py
index d41ab885..75051a15 100644
--- a/modules/sfp_whois.py
+++ b/modules/sfp_whois.py
@@ -12,11 +12,13 @@
# -------------------------------------------------------------------------------
import ipwhois
-import whois
+
from netaddr import IPAddress
from spiderfoot import SpiderFootEvent, SpiderFootPlugin
+import whois
+
class sfp_whois(SpiderFootPlugin):
diff --git a/setup.cfg b/setup.cfg
index 918512b0..3e06c196 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -1,27 +1,30 @@
[flake8]
flake8-max-line-length = 120
-max-line-length = 120
-max-complexity = 80
+max-complexity = 60
select = C,E,F,W,I,B,DUO,D,R,A,S,Q0,SIM,SFS
-# Note: B9 and SIM tests should be reviewed and fixed instead of ignored
-ignore = E501 W503 B006 E800 B9 SFS301 SF01 Q000 SIM102 SIM110 SIM111 SIM113 SIM115 SIM118
+# Note: D and SIM tests should be reviewed and fixed instead of ignored
+ignore = E501 W503 B006 E800 SFS301 SF01 Q000 SIM102 SIM113 SIM115 D100 D104
# Note: most of these should be fixed instead of ignored
per-file-ignores =
spiderfoot/event.py:A,D
- spiderfoot/plugin.py:I,D
- spiderfoot/db.py:A,D,SFS101,SIM106
- sf.py:I,D,SFS101,SFS201
- sflib.py:I,D,SFS101
- sfwebui.py:A,I,D,SFS101,Q003
- sfscan.py:I,D,SIM105,SIM109
- modules/*:I,D
+ spiderfoot/plugin.py:D,B902
+ spiderfoot/db.py:D,SFS101,B902
+ spiderfoot/helpers.py:D
+ spiderfoot/target.py:D
+ spiderfoot/logger.py:D
+ sf.py:I,D,SFS201,B902
+ sflib.py:I,D,SFS101,SIM110,SIM111,B902
+ sfwebui.py:A,B902
+ sfscan.py:I,D,SIM105,B902
+ modules/*:D,B902
+ modules/sfp_binaryedge.py:D,B902,C901
modules/sfp__stor_stdout.py:D,SFS201
modules/sfp_bitcoin.py:D,SFS101
modules/sfp_reversewhois.py:D,R504
- modules/sfp_zetalytics.py:D,R504
- spiderfoot/*:I,D
+ modules/sfp_ripe.py:D,SIM110,B902
+ modules/sfp_zetalytics.py:D,R504,B902
spiderfoot/__init__.py:I,D,F401
- sfcli.py:D
+ sfcli.py:D,B902
test/*:I,D,SIM117
docs/conf.py:A,D
diff --git a/sfscan.py b/sfscan.py
index d2e047d8..4f865e62 100644
--- a/sfscan.py
+++ b/sfscan.py
@@ -165,7 +165,7 @@ class SpiderFootScanner():
proxy_port = int(self.__config.get('_socks3port') or 0)
if not proxy_port:
- if proxy_type == '4' or proxy_type == '5':
+ if proxy_type in ['4', '5']:
proxy_port = 1080
elif proxy_type.upper() == 'HTTP':
proxy_port = 8080
diff --git a/sfwebui.py b/sfwebui.py
index 2b4846c8..5b372ec6 100644
--- a/sfwebui.py
+++ b/sfwebui.py
@@ -15,7 +15,6 @@ import json
import logging
import multiprocessing as mp
import random
-import secure
import time
from copy import deepcopy
from io import StringIO
@@ -23,14 +22,19 @@ from operator import itemgetter
import cherrypy
from cherrypy import _cperror
+
from mako.lookup import TemplateLookup
from mako.template import Template
+import secure
+
+from sflib import SpiderFoot
+
+from sfscan import startSpiderFootScanner
+
from spiderfoot import SpiderFootDb
from spiderfoot import SpiderFootHelpers
from spiderfoot import __version__
-from sflib import SpiderFoot
-from sfscan import startSpiderFootScanner
from spiderfoot.logger import logListenerSetup, logWorkerSetup
mp.set_start_method("spawn", force=True)
@@ -46,17 +50,17 @@ class SpiderFootWebUi:
docroot = ''
def __init__(self, web_config, config, loggingQueue=None):
- """Initialize web server
+ """Initialize web server.
Args:
- web_config: config settings for web interface (interface, port, root path)
- config: SpiderFoot config
+ web_config (dict): config settings for web interface (interface, port, root path)
+ config (dict): SpiderFoot config
+ loggingQueue: TBD
Raises:
TypeError: arg type is invalid
ValueError: arg value is invalid
"""
-
if not isinstance(config, dict):
raise TypeError(f"config is {type(config)}; expected dict()")
if not config:
@@ -115,8 +119,7 @@ class SpiderFootWebUi:
})
def error_page(self):
- """Error page"""
-
+ """Error page."""
cherrypy.response.status = 500
if self.config['_debug']:
@@ -125,7 +128,7 @@ class SpiderFootWebUi:
cherrypy.response.body = b"<html><body>Error</body></html>"
def error_page_401(self, status, message, traceback, version):
- """Unauthorized access HTTP 401 error page
+ """Unauthorized access HTTP 401 error page.
Args:
status (str): HTTP response status code and message
@@ -139,7 +142,7 @@ class SpiderFootWebUi:
return ""
def error_page_404(self, status, message, traceback, version):
- """Error page 404
+ """Not found error page 404.
Args:
status (str): HTTP response status code and message
@@ -154,7 +157,7 @@ class SpiderFootWebUi:
return templ.render(message='Not Found', docroot=self.docroot, status=status, version=__version__)
def jsonify_error(self, status, message):
- """JSONify error response.
+ """Jsonify error response.
Args:
status (str): HTTP response status code and message
@@ -163,7 +166,6 @@ class SpiderFootWebUi:
Returns:
str: HTTP response template
"""
-
cherrypy.response.headers['Content-Type'] = 'application/json'
cherrypy.response.status = status
return {
@@ -174,7 +176,7 @@ class SpiderFootWebUi:
}
def error(self, message):
- """Generic error, but not exposed as not called directly
+ """Show generic error page with error message.
Args:
message (str): error message
@@ -182,7 +184,6 @@ class SpiderFootWebUi:
Returns:
None
"""
-
templ = Template(filename='spiderfoot/templates/error.tmpl', lookup=self.lookup)
return templ.render(message=message, docroot=self.docroot, version=__version__)
@@ -198,7 +199,6 @@ class SpiderFootWebUi:
Raises:
TypeError: inputList type was invalid
"""
-
if not isinstance(inputList, list):
raise TypeError(f"inputList is {type(inputList)}; expected list()")
@@ -209,7 +209,7 @@ class SpiderFootWebUi:
ret.append('')
continue
c = html.escape(item, True)
- c = c.replace('\'', '&quot;')
+ c = c.replace("'", '&quot;')
# We don't actually want & translated to &amp;
c = c.replace("&amp;", "&").replace("&quot;", "\"")
ret.append(c)
@@ -217,7 +217,7 @@ class SpiderFootWebUi:
return ret
def searchBase(self, id=None, eventType=None, value=None):
- """Search
+ """Search.
Args:
id: TBD
@@ -274,7 +274,7 @@ class SpiderFootWebUi:
@cherrypy.expose
def scaneventresultexport(self, id, type, dialect="excel"):
- """Get scan event result data in CSV format
+ """Get scan event result data in CSV format.
Args:
id (str): scan ID
@@ -284,7 +284,6 @@ class SpiderFootWebUi:
Returns:
string: results in CSV format
"""
-
dbh = SpiderFootDb(self.config)
data = dbh.scanResultEvent(id, type)
fileobj = StringIO()
@@ -303,7 +302,7 @@ class SpiderFootWebUi:
@cherrypy.expose
def scaneventresultexportmulti(self, ids, dialect="excel"):
- """Get scan event result data in CSV format for multiple scans
+ """Get scan event result data in CSV format for multiple scans.
Args:
ids (str): comma separated list of scan IDs
@@ -312,7 +311,6 @@ class SpiderFootWebUi:
Returns:
string: results in CSV format
"""
-
dbh = SpiderFootDb(self.config)
scaninfo = dict()
data = list()
@@ -337,7 +335,7 @@ class SpiderFootWebUi:
@cherrypy.expose
def scansearchresultexport(self, id, eventType=None, value=None, dialect="excel"):
- """Get search result data in CSV format
+ """Get search result data in CSV format.
Args:
id (str): scan ID
@@ -348,7 +346,6 @@ class SpiderFootWebUi:
Returns:
string: results in CSV format
"""
-
data = self.searchBase(id, eventType, value)
fileobj = StringIO()
parser = csv.writer(fileobj, dialect=dialect)
@@ -367,7 +364,7 @@ class SpiderFootWebUi:
@cherrypy.expose
def scanexportjsonmulti(self, ids):
- """Get scan event result data in JSON format for multiple scans
+ """Get scan event result data in JSON format for multiple scans.
Args:
ids (str): comma separated list of scan IDs
@@ -375,7 +372,6 @@ class SpiderFootWebUi:
Returns:
string: results in CSV format
"""
-
dbh = SpiderFootDb(self.config)
scaninfo = list()
scan_name = ""
@@ -422,7 +418,7 @@ class SpiderFootWebUi:
@cherrypy.expose
def scanviz(self, id, gexf="0"):
- """Export entities from scan results for visualising
+ """Export entities from scan results for visualising.
Args:
id (str): scan ID
@@ -431,7 +427,6 @@ class SpiderFootWebUi:
Returns:
string: GEXF data
"""
-
if not id:
return None
@@ -454,7 +449,7 @@ class SpiderFootWebUi:
@cherrypy.expose
def scanvizmulti(self, ids, gexf="1"):
- """Export entities results from multiple scans in GEXF format
+ """Export entities results from multiple scans in GEXF format.
Args:
ids (str): scan IDs
@@ -488,7 +483,7 @@ class SpiderFootWebUi:
@cherrypy.expose
@cherrypy.tools.json_out()
def scanopts(self, id):
- """Configuration used for a scan
+ """Return configuration used for the specified scan as JSON.
Args:
id: scan ID
@@ -535,7 +530,7 @@ class SpiderFootWebUi:
@cherrypy.expose
def rerunscan(self, id):
- """Rerun a scan
+ """Rerun a scan.
Args:
id (str): scan ID
@@ -546,7 +541,6 @@ class SpiderFootWebUi:
Raises:
HTTPRedirect: redirect to info page for new scan
"""
-
# Snapshot the current configuration to be used by the scan
cfg = deepcopy(self.config)
modlist = list()
@@ -596,7 +590,7 @@ class SpiderFootWebUi:
@cherrypy.expose
def rerunscanmulti(self, ids):
- """Rerun scans
+ """Rerun scans.
Args:
ids (str): comma separated list of scan IDs
@@ -604,7 +598,6 @@ class SpiderFootWebUi:
Returns:
None
"""
-
# Snapshot the current configuration to be used by the scan
cfg = deepcopy(self.config)
modlist = list()
@@ -652,12 +645,11 @@ class SpiderFootWebUi:
@cherrypy.expose
def newscan(self):
- """Configure a new scan
+ """Configure a new scan.
Returns:
None
"""
-
dbh = SpiderFootDb(self.config)
types = dbh.eventTypes()
templ = Template(filename='spiderfoot/templates/newscan.tmpl', lookup=self.lookup)
@@ -668,7 +660,7 @@ class SpiderFootWebUi:
@cherrypy.expose
def clonescan(self, id):
"""
- Clone an existing scan (pre-selected options in the newscan page)
+ Clone an existing scan (pre-selected options in the newscan page).
Args:
id (str): scan ID to clone
@@ -676,7 +668,6 @@ class SpiderFootWebUi:
Returns:
None
"""
-
dbh = SpiderFootDb(self.config)
types = dbh.eventTypes()
info = dbh.scanInstanceGet(id)
@@ -707,18 +698,17 @@ class SpiderFootWebUi:
@cherrypy.expose
def index(self):
- """Main page listing scans available
+ """Show scan list page.
Returns:
None
"""
-
templ = Template(filename='spiderfoot/templates/scanlist.tmpl', lookup=self.lookup)
return templ.render(pageid='SCANLIST', docroot=self.docroot, version=__version__)
@cherrypy.expose
def scaninfo(self, id):
- """Information about a selected scan
+ """Information about a selected scan.
Args:
id (str): scan id
@@ -726,7 +716,6 @@ class SpiderFootWebUi:
Returns:
None
"""
-
dbh = SpiderFootDb(self.config)
res = dbh.scanInstanceGet(id)
if res is None:
@@ -738,7 +727,7 @@ class SpiderFootWebUi:
@cherrypy.expose
def opts(self, updated=None):
- """Settings
+ """Show module and global settings page.
Args:
updated: TBD
@@ -746,7 +735,6 @@ class SpiderFootWebUi:
Returns:
None
"""
-
templ = Template(filename='spiderfoot/templates/opts.tmpl', lookup=self.lookup)
self.token = random.SystemRandom().randint(0, 99999999)
return templ.render(opts=self.config, pageid='SETTINGS', token=self.token, version=__version__,
@@ -754,7 +742,7 @@ class SpiderFootWebUi:
@cherrypy.expose
def optsexport(self, pattern=None):
- """Export configuration
+ """Export configuration.
Args:
pattern: TBD
@@ -762,7 +750,6 @@ class SpiderFootWebUi:
Returns:
None
"""
-
sf = SpiderFoot(self.config)
conf = sf.configSerialize(self.config)
content = ""
@@ -773,9 +760,9 @@ class SpiderFootWebUi:
if pattern:
if pattern in opt:
- content += "%s=%s\n" % (opt, conf[opt])
+ content += f"{opt}={conf[opt]}\n"
else:
- content += "%s=%s\n" % (opt, conf[opt])
+ content += f"{opt}={conf[opt]}\n"
cherrypy.response.headers['Content-Disposition'] = 'attachment; filename="SpiderFoot.cfg"'
cherrypy.response.headers['Content-Type'] = "text/plain"
@@ -784,7 +771,7 @@ class SpiderFootWebUi:
@cherrypy.expose
@cherrypy.tools.json_out()
def optsraw(self):
- """Settings
+ """Return global and module settings as json.
Returns:
str: settings as JSON
@@ -808,7 +795,7 @@ class SpiderFootWebUi:
@cherrypy.expose
@cherrypy.tools.json_out()
def scandelete(self, id):
- """Delete scan(s)
+ """Delete scan(s).
Args:
id (str): comma separated list of scan IDs
@@ -837,7 +824,7 @@ class SpiderFootWebUi:
@cherrypy.expose
def savesettings(self, allopts, token, configFile=None):
- """Save settings, also used to completely reset them to default
+ """Save settings, also used to completely reset them to default.
Args:
allopts: TBD
@@ -850,9 +837,8 @@ class SpiderFootWebUi:
Raises:
HTTPRedirect: redirect to scan settings
"""
-
if str(token) != str(self.token):
- return self.error("Invalid token (%s)" % self.token)
+ return self.error(f"Invalid token ({self.token})")
if configFile: # configFile seems to get set even if a file isn't uploaded
if configFile.file:
@@ -875,7 +861,7 @@ class SpiderFootWebUi:
allopts = json.dumps(tmp).encode('utf-8')
except Exception as e:
- return self.error("Failed to parse input file. Was it generated from SpiderFoot? (%s)" % e)
+ return self.error(f"Failed to parse input file. Was it generated from SpiderFoot? ({e})")
# Reset config to default
if allopts == "RESET":
@@ -899,13 +885,13 @@ class SpiderFootWebUi:
self.config = sf.configUnserialize(cleanopts, currentopts)
dbh.configSet(sf.configSerialize(self.config))
except Exception as e:
- return self.error("Processing one or more of your inputs failed: %s" % e)
+ return self.error(f"Processing one or more of your inputs failed: {e}")
raise cherrypy.HTTPRedirect(f"{self.docroot}/opts?updated=1")
@cherrypy.expose
def savesettingsraw(self, allopts, token):
- """Save settings, also used to completely reset them to default
+ """Save settings, also used to completely reset them to default.
Args:
allopts: TBD
@@ -914,11 +900,10 @@ class SpiderFootWebUi:
Returns:
str: save success as JSON
"""
-
cherrypy.response.headers['Content-Type'] = "application/json; charset=utf-8"
if str(token) != str(self.token):
- return json.dumps(["ERROR", "Invalid token (%s)." % self.token]).encode('utf-8')
+ return json.dumps(["ERROR", f"Invalid token ({self.token})."]).encode('utf-8')
# Reset config to default
if allopts == "RESET":
@@ -942,7 +927,7 @@ class SpiderFootWebUi:
self.config = sf.configUnserialize(cleanopts, currentopts)
dbh.configSet(sf.configSerialize(self.config))
except Exception as e:
- return json.dumps(["ERROR", "Processing one or more of your inputs failed: %s" % e]).encode('utf-8')
+ return json.dumps(["ERROR", f"Processing one or more of your inputs failed: {e}"]).encode('utf-8')
return json.dumps(["SUCCESS", ""]).encode('utf-8')
@@ -952,7 +937,6 @@ class SpiderFootWebUi:
Returns:
bool: success
"""
-
try:
dbh = SpiderFootDb(self.config)
dbh.configClear() # Clear it in the DB
@@ -964,7 +948,7 @@ class SpiderFootWebUi:
@cherrypy.expose
def resultsetfp(self, id, resultids, fp):
- """Set a bunch of results (hashes) as false positive
+ """Set a bunch of results (hashes) as false positive.
Args:
id (str): scan ID
@@ -974,7 +958,6 @@ class SpiderFootWebUi:
Returns:
str: set false positive status as JSON
"""
-
cherrypy.response.headers['Content-Type'] = "application/json; charset=utf-8"
dbh = SpiderFootDb(self.config)
@@ -990,7 +973,7 @@ class SpiderFootWebUi:
# Cannot set FPs if a scan is not completed
status = dbh.scanInstanceGet(id)
if not status:
- return self.error("Invalid scan ID: %s" % id)
+ return self.error(f"Invalid scan ID: {id}")
if status[5] not in ["ABORTED", "FINISHED", "ERROR-FAILED"]:
return json.dumps([
@@ -1027,7 +1010,6 @@ class SpiderFootWebUi:
Returns:
str: list of event types
"""
-
cherrypy.response.headers['Content-Type'] = "application/json; charset=utf-8"
dbh = SpiderFootDb(self.config)
@@ -1047,7 +1029,6 @@ class SpiderFootWebUi:
Returns:
str: list of modules
"""
-
cherrypy.response.headers['Content-Type'] = "application/json; charset=utf-8"
modinfo = list(self.config['__modules__'].keys())
@@ -1099,7 +1080,7 @@ class SpiderFootWebUi:
@cherrypy.expose
def startscan(self, scanname, scantarget, modulelist, typelist, usecase):
- """Initiate a scan
+ """Initiate a scan.
Args:
scanname (str): scan name
@@ -1232,7 +1213,7 @@ class SpiderFootWebUi:
@cherrypy.expose
@cherrypy.tools.json_out()
def stopscan(self, id):
- """Stop a scan
+ """Stop a scan.
Args:
id (str): comma separated list of scan IDs
@@ -1274,10 +1255,10 @@ class SpiderFootWebUi:
@cherrypy.expose
@cherrypy.tools.json_out()
def scanlog(self, id, limit=None, rowId=None, reverse=None):
- """Scan log data
+ """Scan log data.
Args:
- id: TBD
+ id (str): scan ID
limit: TBD
rowId: TBD
reverse: TBD
@@ -1357,7 +1338,7 @@ class SpiderFootWebUi:
@cherrypy.expose
@cherrypy.tools.json_out()
def scanstatus(self, id):
- """Basic information about a scan.
+ """Show basic information about a scan, including status and number of each event type.
Args:
id (str): scan ID
@@ -1414,7 +1395,7 @@ class SpiderFootWebUi:
@cherrypy.expose
@cherrypy.tools.json_out()
def scaneventresults(self, id, eventType, filterfp=False):
- """Event results for a scan.
+ """Return all event results for a scan as JSON.
Args:
id (str): scan ID
@@ -1454,7 +1435,7 @@ class SpiderFootWebUi:
@cherrypy.expose
@cherrypy.tools.json_out()
def scaneventresultsunique(self, id, eventType, filterfp=False):
- """Unique event results for a scan.
+ """Return unique event results for a scan as JSON.
Args:
id (str): filter search results by scan ID
diff --git a/spiderfoot/db.py b/spiderfoot/db.py
index 917af80a..132f12a0 100644
--- a/spiderfoot/db.py
+++ b/spiderfoot/db.py
@@ -507,12 +507,11 @@ class SpiderFootDb:
))
self.conn.commit()
except sqlite3.Error as e:
- if "locked" in e.args[0] or "thread" in e.args[0]:
- # print("[warning] Couldn't log due to SQLite limitations. You can probably ignore this.")
- # log.critical(f"Unable to log event in DB due to lock: {e.args[0]}")
- pass
- else:
+ if "locked" not in e.args[0] and "thread" not in e.args[0]:
raise IOError(f"Unable to log scan event in DB: {e.args[0]}")
+ # print("[warning] Couldn't log due to SQLite limitations. You can probably ignore this.")
+ # log.critical(f"Unable to log event in DB due to lock: {e.args[0]}")
+ pass
def scanInstanceCreate(self, instanceId, scanName, scanTarget):
"""Store a scan instance in the database.
@@ -1000,7 +999,7 @@ class SpiderFootDb:
except sqlite3.Error as e:
raise IOError(f"Unable to clear configuration from the database: {e.args[0]}")
- def scanConfigSet(self, id, optMap=dict()):
+ def scanConfigSet(self, scan_id, optMap=dict()):
"""Store a configuration value for a scan.
Args:
@@ -1026,10 +1025,10 @@ class SpiderFootDb:
# Module option
if ":" in opt:
parts = opt.split(':')
- qvals = [id, parts[0], parts[1], optMap[opt]]
+ qvals = [scan_id, parts[0], parts[1], optMap[opt]]
else:
# Global option
- qvals = [id, "GLOBAL", opt, optMap[opt]]
+ qvals = [scan_id, "GLOBAL", opt, optMap[opt]]
try:
self.dbh.execute(qry, qvals)
diff --git a/spiderfoot/helpers.py b/spiderfoot/helpers.py
index 3e787dba..3aace936 100644
--- a/spiderfoot/helpers.py
+++ b/spiderfoot/helpers.py
@@ -2,6 +2,7 @@ import json
import random
import re
import uuid
+
import networkx as nx
from networkx.readwrite.gexf import GEXFWriter
@@ -46,7 +47,7 @@ class SpiderFootHelpers():
@staticmethod
def buildGraphGexf(root, title, data, flt=[]):
- """Convert supplied raw data into GEXF (Graph Exchange XML Format) format (e.g. for Gephi)
+ """Convert supplied raw data into GEXF (Graph Exchange XML Format) format (e.g. for Gephi).
Args:
root (str): TBD
@@ -57,7 +58,6 @@ class SpiderFootHelpers():
Returns:
str: GEXF formatted XML
"""
-
mapping = SpiderFootHelpers.buildGraphData(data, flt)
graph = nx.Graph()
@@ -109,7 +109,6 @@ class SpiderFootHelpers():
Returns:
str: TBD
"""
-
mapping = SpiderFootHelpers.buildGraphData(data, flt)
ret = dict()
ret['nodes'] = list()
@@ -247,7 +246,6 @@ class SpiderFootHelpers():
Returns:
dict: nested tree
"""
-
if not isinstance(data, dict):
# TODO: print error
return {}
@@ -317,7 +315,6 @@ class SpiderFootHelpers():
Returns:
str: scan instance unique ID
"""
-
return str(uuid.uuid4()).split("-")[0].upper()
@staticmethod
@@ -335,7 +332,6 @@ class SpiderFootHelpers():
Fix whitespace parsing; ie, " " is not a valid disallowed path
"""
-
returnArr = list()
if not isinstance(robotsTxtData, str):
diff --git a/spiderfoot/logger.py b/spiderfoot/logger.py
index c7a554ae..dd08f717 100644
--- a/spiderfoot/logger.py
+++ b/spiderfoot/logger.py
@@ -1,15 +1,16 @@
-import sys
import logging
-from pathlib import Path
+import sys
from contextlib import suppress
from logging.handlers import QueueHandler, QueueListener
+from pathlib import Path
+
from spiderfoot import SpiderFootDb
log_format = logging.Formatter("%(asctime)s [%(levelname)s] %(message)s")
class SpiderFootSqliteLogHandler(logging.Handler):
- """Handler for logging to SQLite database
+ """Handler for logging to SQLite database.
This ensure all sqlite logging is done from a single
process and a single database handle.
@@ -40,6 +41,7 @@ class SpiderFootSqliteLogHandler(logging.Handler):
def logListenerSetup(loggingQueue, opts=None):
"""Create and start a SpiderFoot log listener in its own thread.
+
This function should be called as soon as possible in the main
process, or whichever process is attached to stdin/stdout.
diff --git a/spiderfoot/plugin.py b/spiderfoot/plugin.py
index b32f4521..673d7525 100644
--- a/spiderfoot/plugin.py
+++ b/spiderfoot/plugin.py
@@ -1,6 +1,6 @@
import logging
-import threading
import queue
+import threading
from time import sleep
diff --git a/spiderfoot/target.py b/spiderfoot/target.py
index 5494df63..238aef2a 100644
--- a/spiderfoot/target.py
+++ b/spiderfoot/target.py
@@ -25,7 +25,6 @@ class SpiderFootTarget():
targetValue (str): target value
typeName (str): target type
"""
-
self.targetType = typeName
self.targetValue = targetValue
self.targetAliases = list()
@@ -102,7 +101,6 @@ class SpiderFootTarget():
Returns:
list: target aliases
"""
-
ret = list()
for item in self.targetAliases:
if item['type'] == typeName:
@@ -115,7 +113,6 @@ class SpiderFootTarget():
Returns:
list: domains associated with the target
"""
-
e = self._getEquivalents("INTERNET_NAME")
if self.targetType in ["INTERNET_NAME", "EMAILADDR"] and self.targetValue.lower() not in e:
e.append(self.targetValue.lower())
@@ -135,7 +132,6 @@ class SpiderFootTarget():
Returns:
list: TBD
"""
-
e = self._getEquivalents("IP_ADDRESS")
if self.targetType == "IP_ADDRESS":
e.append(self.targetValue)
@@ -147,8 +143,7 @@ class SpiderFootTarget():
return e
def matches(self, value, includeParents=False, includeChildren=True):
- """Check whether the supplied value is "tightly" related
- to the original target.
+ """Check whether the supplied value is "tightly" related to the original target.
Tightly in this case means: