SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.
Python attacksurfacecticybersecurityfootprintinghackinginformation-gatheringinformation-securityinfosecintelligence-gatheringosintosint-frameworkosint-reconnaissanceosint-toolpentestingpythonreconsecurity-toolsthreat-intelligencethreatintel
tests: resolve flake8-return violations
Brendan Coles4b6c5f4dbb8f
diff --git a/setup.cfg b/setup.cfgindex e7cbec6d..56124295 100644--- a/setup.cfg+++ b/setup.cfg@@ -1,14 +1,15 @@[flake8]flake8-max-line-length = 120max-complexity = 100-select = C,E,F,W,B,DUO,D-ignore = E501 W503 B006 DUO130 R E800+select = C,E,F,W,B,DUO,D,R+ignore = E501 W503 B006 DUO130 E800 R501 R502per-file-ignores =spiderfoot/__init__.py:F401sfcli.py:Dsflib.py:Dsfscan.py:Dsfwebui.py:D+ modules/*:R[darglint]docstring_style=googlediff --git a/sfcli.py b/sfcli.pyindex 197094fe..6df7c9b9 100755--- a/sfcli.py+++ b/sfcli.py@@ -171,11 +171,14 @@ class SpiderFootCli(cmd.Cmd):val = "0"else:val = "1"+if self.ownopts['cli.spool_file']:return self.do_set("cli.spool = " + val)else:self.edprint("You haven't set cli.spool_file. Set that before enabling spooling.")+ return None+def do_history(self, line):"""history [-l]Short-cut command for set cli.history = 1/0.@@ -871,7 +874,7 @@ class SpiderFootCli(cmd.Cmd):c = self.myparseline(line)if len(c[0]) < 3:self.edprint("Invalid syntax.")- return+ return Noneif "-m" in c[0]:mods = c[0][c[0].index("-m") + 1]@@ -886,7 +889,7 @@ class SpiderFootCli(cmd.Cmd):if not mods and not types and not usecase:self.edprint("Invalid syntax.")- return+ return Nonetarget = c[0][0]if "-n" in c[0]:@@ -905,7 +908,7 @@ class SpiderFootCli(cmd.Cmd):d = self.request(self.ownopts['cli.server_baseurl'] + "/startscan",post=post)if not d:- return+ return Nones = json.loads(d)if s[0] == "SUCCESS":@@ -917,6 +920,8 @@ class SpiderFootCli(cmd.Cmd):if "-w" in c[0]:return self.do_logs(s[1] + " -w")+ return None+# Stop a running scan.def do_stop(self, line):"""stop <sid>diff --git a/sflib.py b/sflib.pyindex 5514fd79..3b6aca52 100644--- a/sflib.py+++ b/sflib.py@@ -156,7 +156,7 @@ class SpiderFoot:return None- def optValueToData(self, val, fatal=True, splitLines=True):+ def optValueToData(self, val):"""Supplied an option value, return the data based on what thevalue is. If val is a URL, you'll get back the fetched content,if val is a file path it will be loaded and get back the contents,@@ -165,38 +165,24 @@ class SpiderFoot:Args:val (str): TBDfatal (bool): TBD- splitLines (bool): TBDReturns:str: TBD"""if not isinstance(val, str):- if fatal:- self.error(f"Invalid option value {val}")- else:- self.error(f"Invalid option value {val}", False)- return None+ self.error(f"Invalid option value {val}")+ return Noneif val.startswith('@'):fname = val.split('@')[1]- try:- self.info("Loading configuration data from: " + fname)- f = open(fname, "r")-- if splitLines:- arr = f.readlines()- ret = list()- for x in arr:- ret.append(x.rstrip('\n'))- else:- ret = f.read()+ self.info(f"Loading configuration data from: {fname}")- f.close()- return ret- except BaseException as e:- if fatal:- self.error(f"Unable to open option file, {fname}: {e}")+ try:+ with open(fname, "r") as f:+ return f.read()+ except Exception as e:+ self.error(f"Unable to open option file, {fname}: {e}")return Noneif val.lower().startswith('http://') or val.lower().startswith('https://'):@@ -205,13 +191,9 @@ class SpiderFoot:session = self.getSession()res = session.get(val)- if splitLines:- return res.content.decode('utf-8').splitlines()-return res.content.decode('utf-8')except BaseException as e:- if fatal:- self.error(f"Unable to open option URL, {val}: {e}")+ self.error(f"Unable to open option URL, {val}: {e}")return Nonereturn val@@ -647,8 +629,7 @@ class SpiderFoot:if mtime > time.time() - timeoutHrs * 3600 or timeoutHrs == 0:with open(cacheFile, "r") as fp:- fileContents = fp.read()- return fileContents+ return fp.read()except BaseException:return None@@ -813,8 +794,6 @@ class SpiderFoot:if not target:return None- targetType = None-regexToType = [{r"^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$": "IP_ADDRESS"},{r"^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}/\d+$": "NETBLOCK_OWNER"},@@ -828,13 +807,13 @@ class SpiderFoot:{r"^([13][a-km-zA-HJ-NP-Z1-9]{25,34})$": "BITCOIN_ADDRESS"}]- # Parse the target and set the targetType+ # Parse the target and set the target typefor rxpair in regexToType:rx = list(rxpair.keys())[0]if re.match(rx, target, re.IGNORECASE | re.UNICODE):- targetType = list(rxpair.values())[0]- break- return targetType+ return list(rxpair.values())[0]++ return Nonedef modulesProducing(self, events):"""Return an array of modules that produce the list of types supplied.@@ -1471,19 +1450,21 @@ class SpiderFoot:list: IP addresses"""- addrs = list()-if not host:- self.error("Unable to resolve %s (Invalid host)" % host, False)- return addrs+ self.error(f"Unable to resolve host: {host} (Invalid host)")+ return list()try:addrs = self.normalizeDNS(socket.gethostbyname_ex(host))except BaseException as e:- self.debug("Unable to resolve %s (%s)" % (host, e))+ self.debug(f"Unable to resolve host: {host} ({e})")+ return list()- if len(addrs):- self.debug("Resolved %s to: %s" % (host, addrs))+ if not addrs:+ self.debug(f"Unable to resolve host: {host}")+ return list()++ self.debug(f"Resolved {host} to: {addrs}")return list(set(addrs))@@ -1497,21 +1478,23 @@ class SpiderFoot:list: list of domain names"""- addrs = list()-if not self.validIP(ipaddr) and not self.validIP6(ipaddr):- self.error("Unable to resolve %s (Invalid IP address)" % ipaddr, False)- return addrs+ self.error(f"Unable to reverse resolve {ipaddr} (Invalid IP address)")+ return list()- self.debug("Performing reverse-resolve of %s" % ipaddr)+ self.debug(f"Performing reverse resolve of {ipaddr}")try:addrs = self.normalizeDNS(socket.gethostbyaddr(ipaddr))except BaseException as e:- self.debug("Unable to resolve %s (%s)" % (ipaddr, e))+ self.debug(f"Unable to reverse resolve IP address: {ipaddr} ({e})")+ return list()- if len(addrs):- self.debug("Resolved %s to: %s" % (ipaddr, addrs))+ if not addrs:+ self.debug(f"Unable to reverse resolve IP address: {ipaddr}")+ return list()++ self.debug(f"Reverse resolved {ipaddr} to: {addrs}")return list(set(addrs))@@ -2720,13 +2703,13 @@ class SpiderFoot:)if response['code'] != '200':- self.error("Failed to get a valid response from the Google API", exception=False)+ self.error("Failed to get a valid response from the Google API")return Nonetry:response_json = json.loads(response['content'])except ValueError:- self.error("The key 'content' in the Google API response doesn't contain valid JSON.", exception=False)+ self.error("The key 'content' in the Google API response doesn't contain valid JSON.")return Noneif "items" not in response_json:@@ -2744,13 +2727,12 @@ class SpiderFoot:search_string=searchString.replace(" ", "%20"),params=urllib.parse.urlencode(params))- results = {++ return {"urls": [str(k['link']) for k in response_json['items']],"webSearchUrl": search_url,}- return results-def bingIterate(self, searchString, opts={}):"""Request search results from the Bing API.@@ -2788,13 +2770,13 @@ class SpiderFoot:)if response['code'] != '200':- self.error("Failed to get a valid response from the bing API", exception=False)+ self.error("Failed to get a valid response from the bing API")return Nonetry:response_json = json.loads(response['content'])except ValueError:- self.error("The key 'content' in the bing API response doesn't contain valid JSON.", exception=False)+ self.error("The key 'content' in the bing API response doesn't contain valid JSON.")return Noneif ("webPages" in response_json and "value" in response_json["webPages"] and "webSearchUrl" in response_json["webPages"]):diff --git a/test/unit/test_spiderfoot.py b/test/unit/test_spiderfoot.pyindex 9fc20c7a..9a5c9615 100644--- a/test/unit/test_spiderfoot.py+++ b/test/unit/test_spiderfoot.py@@ -103,28 +103,28 @@ class TestSpiderFoot(unittest.TestCase):res = sf.refreshTorIdent()self.assertEqual(None, res)- def test_opt_value_to_data(self):+ def test_optValueToData_should_return_data_as_string(self):"""- Test optValueToData(self, val, fatal=True, splitLines=True)+ Test optValueToData(self, val)"""sf = SpiderFoot(self.default_options)test_string = "example string"- opt_data = sf.optValueToData(test_string, fatal=False, splitLines=True)+ opt_data = sf.optValueToData(test_string)self.assertIsInstance(opt_data, str)self.assertEqual(test_string, opt_data)- def test_opt_value_to_data_invalid_opt_should_return_none(self):+ def test_optValueToData_argument_val_invalid_type_should_return_None(self):"""- Test optValueToData(self, val, fatal=True, splitLines=True)+ Test optValueToData(self, val)"""sf = SpiderFoot(self.default_options)- opt_data = sf.optValueToData(None, fatal=False, splitLines=True)- self.assertEqual(None, opt_data)-- opt_data = sf.optValueToData([], fatal=False, splitLines=True)- self.assertEqual(None, opt_data)+ invalid_types = [None, list(), int(), dict()]+ for invalid_type in invalid_types:+ with self.subTest(invalid_type=invalid_type):+ opt_data = sf.optValueToData(invalid_type)+ self.assertEqual(opt_data, None)def test_buildGraphData_should_return_a_set(self):"""