SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.
Python attacksurfacecticybersecurityfootprintinghackinginformation-gatheringinformation-securityinfosecintelligence-gatheringosintosint-frameworkosint-reconnaissanceosint-toolpentestingpythonreconsecurity-toolsthreat-intelligencethreatintel
sflib: Update Event / Target error handling, docs, tests
Brendan Colesc38c10f476af
diff --git a/sflib.py b/sflib.pyindex 6d31c55a..d12b7e20 100644--- a/sflib.py+++ b/sflib.py@@ -2475,13 +2475,18 @@ class SpiderFootPlugin(object):self.__outputFilter__ = typesdef tempStorage(self):- """Module temporary storage.+ """For future use. Module temporary storage.++ A dictionary used to persist state (in memory) for a module.++ Todo:+ Move all module state to use this, which then would enable a scan to be paused/resumed.Note:- For SpiderFoot HX compatability of modules.+ Required for SpiderFoot HX compatability of modules.Returns:- dict: TB+ dict: module temporary state data"""return dict()@@ -2635,8 +2640,9 @@ class SpiderFootPlugin(object):return None-# Class for targetsclass SpiderFootTarget(object):+ """SpiderFoot target."""+_validTypes = ["IP_ADDRESS", 'IPV6_ADDRESS', "NETBLOCK_OWNER", "INTERNET_NAME","EMAILADDR", "HUMAN_NAME", "BGP_AS_OWNER", 'PHONE_NUMBER', "USERNAME"]targetType = None@@ -2644,16 +2650,27 @@ class SpiderFootTarget(object):targetAliases = list()def __init__(self, targetValue, typeName):+ """Initialize SpiderFoot target.++ Args:+ targetValue (str): target value+ typeName (str): target type++ Returns:+ None++ Raises:+ TypeError: targetValue type was invalid+ ValueError: typeName value was an invalid target type+ """+if not isinstance(targetValue, str):raise TypeError("Invalid target value %s; expected %s" % type(targetValue))if typeName not in self._validTypes:raise ValueError("Invalid target type %s; expected %s" % (typeName, self._validTypes))self.targetType = typeName- if isinstance(targetValue, str):- self.targetValue = targetValue- else:- self.targetValue = str(targetValue).lower()+ self.targetValue = targetValueself.targetAliases = list()def getType(self):@@ -2662,13 +2679,22 @@ class SpiderFootTarget(object):def getValue(self):return self.targetValue- # Specify other hostnames, IPs, etc. that are aliases for- # this target.- # For instance, if the user searched for an ASN, a module- # might supply all the nested subnets as aliases.- # Or, if a user searched for an IP address, a module- # might supply the hostname as an alias.def setAlias(self, value, typeName):+ """Specify other hostnames, IPs, etc. that are aliases for this target.++ For instance, if the user searched for an ASN, a module+ might supply all the nested subnets as aliases.+ Or, if a user searched for an IP address, a module+ might supply the hostname as an alias.++ Args:+ value (str): TBD+ typeName (str): TBD++ Returns:+ None+ """+if value is None:return@@ -2680,17 +2706,34 @@ class SpiderFootTarget(object):)def getAliases(self):+ """TBD++ Returns:+ list: target aliases+ """+return self.targetAliasesdef _getEquivalents(self, typeName):+ """TBD++ Returns:+ list: target aliases+ """+ret = list()for item in self.targetAliases:if item['type'] == typeName:ret.append(item['value'].lower())return ret- # Get all domains associated with the targetdef getNames(self):+ """Get all domains associated with the target.++ Returns:+ list: domains associated with the target+ """+e = self._getEquivalents("INTERNET_NAME")if self.targetType in ["INTERNET_NAME", "EMAILADDR"] and self.targetValue.lower() not in e:e.append(self.targetValue.lower())@@ -2698,36 +2741,50 @@ class SpiderFootTarget(object):names = list()for name in e:names.append(name.decode("utf-8") if type(name) == bytes else name)+return names- # Get all IP Subnets or IP Addresses associated with the targetdef getAddresses(self):+ """Get all IP Subnets or IP Addresses associated with the target.++ Returns:+ list: TBD+ """+e = self._getEquivalents("IP_ADDRESS")if self.targetType == "IP_ADDRESS":e.append(self.targetValue)+e = self._getEquivalents("IPV6_ADDRESS")if self.targetType == "IPV6_ADDRESS":e.append(self.targetValue)+return e- # Check whether the supplied value is "tightly" related- # to the original target.- # Tightly in this case means:- # 1. If the value is an IP:- # 1.1 is it in the list of aliases or the target itself?- # 1.2 is it on the target's subnet?- # 2. If the value is a name (subdomain, domain, hostname):- # 2.1 is it in the list of aliases or the target itself?- # 2.2 is it a parent of the aliases of the target (domain/subdomain)- # 2.3 is it a child of the aliases of the target (hostname)- # Arguments:- # * value can be an Internet Name (hostname, subnet, domain)- # or an IP address.- # * includeParents = True means you consider a value that is- # a parent domain of the target to still be a tight relation.- # * includeChildren = False means you don't consider a value- # that is a child of the target to be a tight relation.def matches(self, value, includeParents=False, includeChildren=True):+ """Check whether the supplied value is "tightly" related+ to the original target.++ Tightly in this case means:+ 1. If the value is an IP:+ 1.1 is it in the list of aliases or the target itself?+ 1.2 is it on the target's subnet?+ 2. If the value is a name (subdomain, domain, hostname):+ 2.1 is it in the list of aliases or the target itself?+ 2.2 is it a parent of the aliases of the target (domain/subdomain)+ 2.3 is it a child of the aliases of the target (hostname)++ Args:+ value (str): can be an Internet Name (hostname, subnet, domain) or an IP address.+ includeParents (bool): True means you consider a value that is+ a parent domain of the target to still be a tight relation.+ includeChildren (bool): False means you don't consider a value+ that is a child of the target to be a tight relation.++ Returns:+ bool: whether the value matches the target+ """+if value is None:return False@@ -2770,8 +2827,9 @@ class SpiderFootTarget(object):return None-# Class for SpiderFoot Eventsclass SpiderFootEvent(object):+ """SpiderFoot event."""+generated = NoneeventType = Noneconfidence = None@@ -2787,6 +2845,24 @@ class SpiderFootEvent(object):def __init__(self, eventType, data, module, sourceEvent,confidence=100, visibility=100, risk=0):+ """Initialize SpiderFoot event.++ Args:+ eventType (str): event type+ data (str): event data+ module (str): module from which the event originated+ sourceEvent (SpiderFootEvent): source event+ confidence (int): event confidence+ visibility (int): event visibility+ risk (int): event risk++ Returns:+ dict: event as dict++ Raises:+ TypeError: arg type was invalid+ """+self.eventType = eventTypeself.generated = time.time()self.confidence = confidence@@ -2795,16 +2871,13 @@ class SpiderFootEvent(object):self.module = moduleself.sourceEvent = sourceEvent- if type(data) != str and type(data) != str:+ if not isinstance(data, str):print("FATAL: Only string events are accepted, not '%s'." % type(data))print("FATAL: Offending module: %s" % module)print("FATAL: Offending type: %s" % eventType)- sys.exit(-1)+ raise TypeError("data is %s; expected str()" % type(data))- if type(data) != str and data != None:- self.data = str(data)- else:- self.data = data+ self.data = data# "ROOT" is a special "hash" reserved for elements with no# actual parent (e.g. the first page spidered.)@@ -2812,17 +2885,23 @@ class SpiderFootEvent(object):self.sourceEventHash = "ROOT"return- if type(sourceEvent) != SpiderFootEvent:+ if not isinstance(sourceEvent, SpiderFootEvent):print("FATAL: Invalid source event: %s" % sourceEvent)print("FATAL: Offending module: %s" % module)print("FATAL: Offending type: %s" % eventType)- sys.exit(-1)+ raise TypeError("sourceEvent is %s; expected SpiderFootEvent()" % type(sourceEvent))self.sourceEventHash = sourceEvent.getHash()self.__id = self.eventType + str(self.generated) + self.module + \str(random.SystemRandom().randint(0, 99999999))def asDict(self):+ """Return event as dictionary.++ Returns:+ dict: event as dictionary+ """+evt_dict = {'generated': int(self.generated),'type': self.eventType,@@ -2837,22 +2916,76 @@ class SpiderFootEvent(object):return evt_dict- # Unique hash of this eventdef getHash(self):+ """ Unique hash of this event.++ Returns:+ str: unique SHA256 hash of the event, or "ROOT"+ """+if self.eventType == "ROOT":return "ROOT"+digestStr = self.__id.encode('raw_unicode_escape')return hashlib.sha256(digestStr).hexdigest()- # Update variables as new information becomes availabledef setConfidence(self, confidence):+ """Update event confidence attribute as new information becomes available.++ Args:+ confidence (int): confidence (0 to 100)++ Raises:+ TypeError: confidence type was invalid+ ValueError: confidence value was invalid+ """+ if not isinstance(confidence, int):+ raise TypeError("confidence is %s; expected int()" % type(confidence))+ if confidence > 100 or confidence < 0:+ raise ValueError("Invalid confidence: %s. Must be between 0 and 100" % confidence)+self.confidence = confidencedef setVisibility(self, visibility):+ """Update event visibility attribute.++ Args:+ visibility (int): visibility (0 to 100)++ Raises:+ TypeError: visibility type was invalid+ ValueError: visibility value was invalid+ """+ if not isinstance(visibility, int):+ raise TypeError("visibility is %s; expected int()" % type(visibility))+ if visibility > 100 or visibility < 0:+ raise ValueError("Invalid visibility: %s. Must be between 0 and 100" % visibility)+self.visibility = visibilitydef setRisk(self, risk):+ """Update event risk attribute.++ Args:+ risk (int): risk (0 to 100)++ Raises:+ TypeError: risk type was invalid+ ValueError: risk value was invalid+ """+ if not isinstance(risk, int):+ raise TypeError("risk is %s; expected int()" % type(risk))+ if risk > 100 or risk < 0:+ raise ValueError("Invalid risk: %s. Must be between 0 and 100" % risk)+self.risk = riskdef setSourceEventHash(self, srcHash):+ """Update event source event hash attribute.++ Args:+ srcHash (str): source event hash+ """+self.sourceEventHash = srcHash+diff --git a/test/unit/test_spiderfootevent.py b/test/unit/test_spiderfootevent.pyindex faff99c6..9b5d3bb0 100644--- a/test/unit/test_spiderfootevent.py+++ b/test/unit/test_spiderfootevent.py@@ -34,7 +34,7 @@ class TestSpiderFootEvent(unittest.TestCase):evt = SpiderFootEvent(event_type, event_data, module, source_event)self.assertIsInstance(evt, SpiderFootEvent)- def test_init_invalid_event_data_type_should_exit(self):+ def test_init_invalid_event_data_type_should_raise(self):"""Test __init__(self, eventType, data, module, sourceEvent, confidence=100, visibility=100, risk=0)"""@@ -42,25 +42,25 @@ class TestSpiderFootEvent(unittest.TestCase):module = ''source_event = ''- with self.assertRaises(SystemExit) as cm:- event_data = int(1)- evt = SpiderFootEvent(event_type, event_data, module, source_event)+ invalid_types = [None, list(), dict()]+ for invalid_type in invalid_types:+ with self.subTest(invalid_type=invalid_type):+ with self.assertRaises(TypeError) as cm:+ evt = SpiderFootEvent(event_type, invalid_type, module, source_event)- self.assertEqual(cm.exception.code, -1)-- def test_init_nonroot_event_type_with_no_source_event_should_exit(self):+ def test_init_invalid_source_event_type_should_raise(self):"""Test __init__(self, eventType, data, module, sourceEvent, confidence=100, visibility=100, risk=0)"""- event_data = ''+ event_data = 'example event data'module = ''+ event_type = 'example non-root event type'- with self.assertRaises(SystemExit) as cm:- event_type = 'example non-root event type'- source_event = ''- evt = SpiderFootEvent(event_type, event_data, module, source_event)-- self.assertEqual(cm.exception.code, -1)+ invalid_types = [None, "", list(), dict()]+ for invalid_type in invalid_types:+ with self.subTest(invalid_type=invalid_type):+ with self.assertRaises(TypeError) as cm:+ evt = SpiderFootEvent(event_type, event_data, module, invalid_type)def test_asdict_root_event_should_return_a_dict(self):"""@@ -121,8 +121,7 @@ class TestSpiderFootEvent(unittest.TestCase):self.assertIsInstance(evt_hash, str)- @unittest.skip("unused function")- def test_set_confidence(self):+ def test_set_confidence_invalid_confidence_should_raise(self):"""Test setConfidence(self, confidence)"""@@ -130,12 +129,34 @@ class TestSpiderFootEvent(unittest.TestCase):event_data = ''module = ''source_event = ''+ evt = SpiderFootEvent(event_type, event_data, module, source_event)+ invalid_types = [None, "", list(), dict()]+ for invalid_type in invalid_types:+ with self.subTest(invalid_type=invalid_type):+ with self.assertRaises(TypeError) as cm:+ evt.setConfidence(invalid_type)++ with self.assertRaises(ValueError) as cm:+ evt.setConfidence(-1)+ with self.assertRaises(ValueError) as cm:+ evt.setConfidence(101)++ def test_set_confidence_should_set_confidence_attribute(self):+ """+ Test setConfidence(self, confidence)+ """+ event_type = 'ROOT'+ event_data = ''+ module = ''+ source_event = ''evt = SpiderFootEvent(event_type, event_data, module, source_event)- evt.setConfidence('confidence')- @unittest.skip("unused function")- def test_set_visibility(self):+ confidence = 100+ evt.setConfidence(confidence)+ self.assertEqual(confidence, evt.confidence)++ def test_set_visibility_invalid_visibility_should_raise(self):"""Test setVisibility(self, visibility)"""@@ -143,12 +164,34 @@ class TestSpiderFootEvent(unittest.TestCase):event_data = ''module = ''source_event = ''+ evt = SpiderFootEvent(event_type, event_data, module, source_event)++ invalid_types = [None, "", list(), dict()]+ for invalid_type in invalid_types:+ with self.subTest(invalid_type=invalid_type):+ with self.assertRaises(TypeError) as cm:+ evt.setVisibility(invalid_type)++ with self.assertRaises(ValueError) as cm:+ evt.setVisibility(-1)+ with self.assertRaises(ValueError) as cm:+ evt.setVisibility(101)+ def test_set_visibility_should_set_visibility_attribute(self):+ """+ Test setVisibility(self, visibility)+ """+ event_type = 'ROOT'+ event_data = ''+ module = ''+ source_event = ''evt = SpiderFootEvent(event_type, event_data, module, source_event)- evt.setVisibility('visibility')- @unittest.skip("unused function")- def test_set_risk(self):+ visibility = 100+ evt.setVisibility(visibility)+ self.assertEqual(visibility, evt.visibility)++ def test_set_risk_invalid_risk_should_raise(self):"""Test setRisk(self, risk)"""@@ -156,12 +199,34 @@ class TestSpiderFootEvent(unittest.TestCase):event_data = ''module = ''source_event = ''+ evt = SpiderFootEvent(event_type, event_data, module, source_event)++ invalid_types = [None, "", list(), dict()]+ for invalid_type in invalid_types:+ with self.subTest(invalid_type=invalid_type):+ with self.assertRaises(TypeError) as cm:+ evt.setRisk(invalid_type)+ with self.assertRaises(ValueError) as cm:+ evt.setRisk(-1)+ with self.assertRaises(ValueError) as cm:+ evt.setRisk(101)++ def test_set_risk_should_set_risk_attribute(self):+ """+ Test setRisk(self, risk)+ """+ event_type = 'ROOT'+ event_data = ''+ module = ''+ source_event = ''evt = SpiderFootEvent(event_type, event_data, module, source_event)- evt.setRisk('risk')- @unittest.skip("unused function")- def test_set_source_event_hash(self):+ risk = 100+ evt.setRisk(risk)+ self.assertEqual(risk, evt.risk)++ def test_set_source_event_hash_should_set_source_event_hash_attribute(self):"""Test setSourceEventHash(self, srcHash)"""@@ -169,9 +234,11 @@ class TestSpiderFootEvent(unittest.TestCase):event_data = ''module = ''source_event = ''-evt = SpiderFootEvent(event_type, event_data, module, source_event)- evt.setSourceEventHash('source event hash')++ source_event_hash = 'source event hash'+ evt.setSourceEventHash(source_event_hash)+ self.assertEqual(source_event_hash, evt.sourceEventHash)if __name__ == '__main__':unittest.main()