SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.
Python attacksurfacecticybersecurityfootprintinghackinginformation-gatheringinformation-securityinfosecintelligence-gatheringosintosint-frameworkosint-reconnaissanceosint-toolpentestingpythonreconsecurity-toolsthreat-intelligencethreatintel
Added ThreatCrowd module.
Steve27cd06131929
diff --git a/modules/sfp_threatcrowd.py b/modules/sfp_threatcrowd.pynew file mode 100644index 00000000..97479326--- /dev/null+++ b/modules/sfp_threatcrowd.py@@ -0,0 +1,179 @@+# -*- coding: utf-8 -*-+# -------------------------------------------------------------------------------+# Name: sfp_threatcrowd+# Purpose: Query threatcrowd.org for identified IP addresses.+#+# Author: Steve Micallef <[email protected]>+#+# Created: 21/11/2016+# Copyright: (c) Steve Micallef+# Licence: GPL+# -------------------------------------------------------------------------------++import json+import time+from netaddr import IPNetwork+from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+++class sfp_threatcrowd(SpiderFootPlugin):+ """ThreatCrowd:Investigate,Passive:Blacklists::Obtain information from ThreatCrowd about identified IP addresses, domains and e-mail addresses."""++ # Default options+ opts = {+ "checkcohosts": True,+ "checkaffiliates": True,+ 'netblocklookup': True,+ 'maxnetblock': 24,+ 'subnetlookup': True,+ 'maxsubnet': 24+ }++ # Option descriptions+ optdescs = {+ "checkcohosts": "Check co-hosted sites?",+ "checkaffiliates": "Check affiliates?",+ 'netblocklookup': "Look up all IPs on netblocks deemed to be owned by your target for possible hosts on the same target subdomain/domain?",+ 'maxnetblock': "If looking up owned netblocks, the maximum netblock size to look up all IPs within (CIDR value, 24 = /24, 16 = /16, etc.)",+ 'subnetlookup': "Look up all IPs on subnets which your target is a part of?",+ 'maxsubnet': "If looking up subnets, the maximum subnet size to look up all the IPs within (CIDR value, 24 = /24, 16 = /16, etc.)"+ }++ # Be sure to completely clear any class variables in setup()+ # or you run the risk of data persisting between scan runs.++ results = dict()+ errorState = False++ def setup(self, sfc, userOpts=dict()):+ self.sf = sfc+ self.results = dict()++ # Clear / reset any other class member variables here+ # or you risk them persisting between threads.++ for opt in userOpts.keys():+ self.opts[opt] = userOpts[opt]++ # What events is this module interested in for input+ def watchedEvents(self):+ return ["IP_ADDRESS", "AFFILIATE_IPADDR", "INTERNET_NAME",+ "CO_HOSTED_SITE", "NETBLOCK_OWNER", "EMAILADDR",+ "NETBLOCK_MEMBER"]++ # What events this module produces+ def producedEvents(self):+ return ["MALICIOUS_IPADDR", "MALICIOUS_INTERNET_NAME",+ "MALICIOUS_COHOST", "MALICIOUS_AFFILIATE_INTERNET_NAME",+ "MALICIOUS_AFFILIATE_IPADDR", "MALICIOUS_NETBLOCK",+ "MALICIOUS_SUBNET", "INTERNET_NAME", "AFFILIATE_INTERNET_NAME",+ "MALICIOUS_EMAILADDR"]++ def query(self, qry):+ ret = None+ url = None++ if self.sf.validIP(qry):+ url = "http://www.threatcrowd.org/searchApi/v2/ip/report/?ip=" + qry++ if "@" in qry:+ url = "http://www.threatcrowd.org/searchApi/v2/email/report/?email=" + qry++ if not url:+ url = "http://www.threatcrowd.org/searchApi/v2/domain/report/?domain=" + qry++ res = self.sf.fetchUrl(url, timeout=self.opts['_fetchtimeout'], useragent="SpiderFoot")++ if res['content'] is None:+ self.sf.info("No ThreatCrowd info found for " + qry)+ return None++ try:+ ret = json.loads(res['content'])+ except Exception as e:+ self.sf.error("Error processing JSON response from ThreatCrowd.", False)+ return None++ return ret++ # Handle events sent to this module+ def handleEvent(self, event):+ eventName = event.eventType+ srcModuleName = event.module+ eventData = event.data++ if self.errorState:+ return None++ self.sf.debug("Received event, " + eventName + ", from " + srcModuleName)++ # Don't look up stuff twice+ if eventData in self.results:+ self.sf.debug("Skipping " + eventData + " as already mapped.")+ return None+ else:+ self.results[eventData] = True++ if eventName.startswith("AFFILIATE") and not self.opts['checkaffiliates']:+ return None++ if eventName == 'CO_HOSTED_SITE' and not self.opts['checkcohosts']:+ return None++ if eventName == 'NETBLOCK_OWNER' and self.opts['netblocklookup']:+ if IPNetwork(eventData).prefixlen < self.opts['maxnetblock']:+ self.sf.debug("Network size bigger than permitted: " ++ str(IPNetwork(eventData).prefixlen) + " > " ++ str(self.opts['maxnetblock']))+ return None++ if eventName == 'NETBLOCK_MEMBER' and self.opts['subnetlookup']:+ if IPNetwork(eventData).prefixlen < self.opts['maxsubnet']:+ self.sf.debug("Network size bigger than permitted: " ++ str(IPNetwork(eventData).prefixlen) + " > " ++ str(self.opts['maxsubnet']))+ return None++ qrylist = list()+ if eventName.startswith("NETBLOCK_"):+ for ipaddr in IPNetwork(eventData):+ qrylist.append(str(ipaddr))+ self.results[str(ipaddr)] = True+ else:+ qrylist.append(eventData)++ for addr in qrylist:+ if self.checkForStop():+ return None++ info = self.query(addr)+ if info is None:+ continue+ if info.get('votes', 0) < 0:+ self.sf.info("Found ThreatCrowd URL data for " + addr)+ if eventName in ["IP_ADDRESS"] or eventName.startswith("NETBLOCK_"):+ evt = "MALICIOUS_IPADDR"++ if eventName == "AFFILIATE_IPADDR":+ evt = "MALICIOUS_AFFILIATE_IPADDR"++ if eventName == "INTERNET_NAME":+ evt = "MALICIOUS_INTERNET_NAME"++ if eventName == "AFFILIATE_INTERNET_NAME":+ evt = "MALICIOUS_AFFILIATE_INTERNET_NAME"++ if eventName == "CO_HOSTED_SITE":+ evt = "MALICIOUS_COHOST"++ if eventName == "EMAILADDR":+ evt = "MALICIOUS_EMAILADDR"++ infourl = "<SFURL>" + info.get('permalink') + "</SFURL>"++ # Notify other modules of what you've found+ e = SpiderFootEvent(evt, "ThreatCrowd [" + addr + "]\n" ++ infourl, self.__name__, event)+ self.notifyListeners(e)++# End of sfp_threatcrowd class