SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.
Python attacksurfacecticybersecurityfootprintinghackinginformation-gatheringinformation-securityinfosecintelligence-gatheringosintosint-frameworkosint-reconnaissanceosint-toolpentestingpythonreconsecurity-toolsthreat-intelligencethreatintel
sfp_abusech: Cleanup (#1317)
bcolesacf803a79d85
diff --git a/modules/sfp_abusech.py b/modules/sfp_abusech.pyindex c017f4e0..be3a123b 100644--- a/modules/sfp_abusech.py+++ b/modules/sfp_abusech.py@@ -1,7 +1,8 @@# -*- coding: utf-8 -*-# --------------------------------------------------------------------------------# Name: sfp_abusech-# Purpose: Checks if an ASN, IP or domain is malicious.+# Name: sfp_abusech+# Purpose: Check if a host/domain, IP address or netblock is malicious according+# to Abuse.ch.## Author: [email protected]#@@ -10,38 +11,16 @@# Licence: GPL# --------------------------------------------------------------------------------import re-from netaddr import IPAddress, IPNetworkfrom spiderfoot import SpiderFootEvent, SpiderFootPlugin-malchecks = {- 'abuse.ch Feodo Tracker (IP)': {- 'id': 'abusefeodoip',- 'checks': ['ip', 'netblock'],- 'url': 'https://feodotracker.abuse.ch/downloads/ipblocklist.txt'- },- 'abuse.ch SSL Blacklist (IP)': {- 'id': 'abusesslblip',- 'checks': ['ip', 'netblock'],- 'url': 'https://sslbl.abuse.ch/blacklist/sslipblacklist.csv',- 'regex': '{0},.*'- },- 'abuse.ch URLhaus (Domain)': {- 'id': 'abuseurlhaus',- 'checks': ['domain'],- 'url': 'https://urlhaus.abuse.ch/downloads/csv_recent/',- 'regex': '.*//{0}/.*'- }-}-class sfp_abusech(SpiderFootPlugin):meta = {'name': "abuse.ch",- 'summary': "Check if a host/domain, IP or netblock is malicious according to abuse.ch.",+ 'summary': "Check if a host/domain, IP address or netblock is malicious according to Abuse.ch.",'flags': [],'useCases': ["Passive", "Investigate"],'categories': ["Reputation Systems"],@@ -49,19 +28,15 @@ class sfp_abusech(SpiderFootPlugin):'website': "https://www.abuse.ch",'model': "FREE_AUTH_UNLIMITED",'references': [- "https://bazaar.abuse.ch/","https://feodotracker.abuse.ch/",- "https://igotphished.abuse.ch/","https://sslbl.abuse.ch/",- "https://urlhaus.abuse.ch/"+ "https://urlhaus.abuse.ch/",],'apiKeyInstructions': ["Visit https://bazaar.abuse.ch/api#api_key","Login using a Twitter Account","Navigate to 'Account Settings'","The API key is listed under 'Your API Key'",- "Visit https://igotphished.abuse.ch/api/",- "Request an API key from the email listed under 'Submit data to I Got Phished'","Visit https://urlhaus.abuse.ch/api/","Login using a Twitter Account at https://urlhaus.abuse.ch/login/","Navigate to https://urlhaus.abuse.ch/api/#account",@@ -101,124 +76,269 @@ class sfp_abusech(SpiderFootPlugin):'checksubnets': "Check if any malicious IPs are found within the same subnet of the target?"}- # Be sure to completely clear any class variables in setup()- # or you run the risk of data persisting between scan runs.-results = None+ errorState = Falsedef setup(self, sfc, userOpts=dict()):self.sf = sfcself.results = self.tempStorage()-- # Clear / reset any other class member variables here- # or you risk them persisting between threads.+ self.errorState = Falsefor opt in list(userOpts.keys()):self.opts[opt] = userOpts[opt]# What events is this module interested in for input- # * = be notified about all events.def watchedEvents(self):- return ["INTERNET_NAME", "IP_ADDRESS",- "NETBLOCK_MEMBER", "AFFILIATE_INTERNET_NAME", "AFFILIATE_IPADDR",- "CO_HOSTED_SITE", "NETBLOCK_OWNER"]+ return [+ "INTERNET_NAME",+ "IP_ADDRESS",+ "NETBLOCK_MEMBER",+ "AFFILIATE_INTERNET_NAME",+ "AFFILIATE_IPADDR",+ "CO_HOSTED_SITE",+ "NETBLOCK_OWNER"+ ]# What events this module produces- # This is to support the end user in selecting modules based on events- # produced.def producedEvents(self):- return ["MALICIOUS_IPADDR", "MALICIOUS_INTERNET_NAME",- "MALICIOUS_AFFILIATE_IPADDR", "MALICIOUS_AFFILIATE_INTERNET_NAME",- "MALICIOUS_SUBNET", "MALICIOUS_COHOST", "MALICIOUS_NETBLOCK"]-- # Look up 'list' type resources- def resourceList(self, replaceme_id, target, targetType):- targetDom = ''- # Get the base domain if we're supplied a domain- if targetType == "domain":- targetDom = self.sf.hostDomain(target, self.opts['_internettlds'])- if not targetDom:- return None-- for check in list(malchecks.keys()):- cid = malchecks[check]['id']- if replaceme_id == cid:- data = dict()- url = malchecks[check]['url']- data['content'] = self.sf.cacheGet("sfmal_" + cid, self.opts.get('cacheperiod', 0))- if data['content'] is None:- data = self.sf.fetchUrl(url, timeout=self.opts['_fetchtimeout'], useragent=self.opts['_useragent'])- if data['content'] is None:- self.sf.error("Unable to fetch " + url)- return None- else:- self.sf.cachePut("sfmal_" + cid, data['content'])-- # If we're looking at netblocks- if targetType == "netblock":- iplist = list()- # Get the regex, replace {0} with an IP address matcher to- # build a list of IP.- # Cycle through each IP and check if it's in the netblock.- if 'regex' in malchecks[check]:- rx = malchecks[check]['regex'].replace("{0}", r"(\d+\.\d+\.\d+\.\d+)")- pat = re.compile(rx, re.IGNORECASE)- self.sf.debug("New regex for " + check + ": " + rx)- for line in data['content'].split('\n'):- grp = re.findall(pat, line)- if len(grp) > 0:- # self.sf.debug("Adding " + grp[0] + " to list.")- iplist.append(grp[0])- else:- iplist = data['content'].split('\n')-- for ip in iplist:- if len(ip) < 8 or ip.startswith("#"):- continue- ip = ip.strip()-- try:- if IPAddress(ip) in IPNetwork(target):- self.sf.debug(f"{ip} found within netblock/subnet {target} in {check}")- return url- except Exception as e:- self.sf.debug(f"Error encountered parsing: {e}")- continue-- return None-- # If we're looking at hostnames/domains/IPs- if 'regex' not in malchecks[check]:- for line in data['content'].split('\n'):- if line == target or (targetType == "domain" and line == targetDom):- self.sf.debug(target + "/" + targetDom + " found in " + check + " list.")- return url- else:- try:- # Check for the domain and the hostname- rxDom = str(malchecks[check]['regex']).format(targetDom)- rxTgt = str(malchecks[check]['regex']).format(target)- for line in data['content'].split('\n'):- if (targetType == "domain" and re.match(rxDom, line, re.IGNORECASE)) or \- re.match(rxTgt, line, re.IGNORECASE):- self.sf.debug(target + "/" + targetDom + " found in " + check + " list.")- return url- except Exception as e:- self.sf.debug("Error encountered parsing 2: " + str(e))- continue-- return None-- def lookupItem(self, resourceId, itemType, target):- for check in list(malchecks.keys()):- cid = malchecks[check]['id']- if cid == resourceId and itemType in malchecks[check]['checks']:- self.sf.debug(f"Checking maliciousness of {target} ({itemType}) with: {cid}")- return self.resourceList(cid, target, itemType)-- return None-- # Handle events sent to this module+ return [+ "MALICIOUS_IPADDR",+ "MALICIOUS_INTERNET_NAME",+ "MALICIOUS_AFFILIATE_IPADDR",+ "MALICIOUS_AFFILIATE_INTERNET_NAME",+ "MALICIOUS_SUBNET",+ "MALICIOUS_COHOST",+ "MALICIOUS_NETBLOCK"+ ]++ def queryFeodoTrackerBlacklist(self, target, targetType):+ blacklist = self.retrieveFeodoTrackerBlacklist()++ if not blacklist:+ return False++ if targetType == "ip":+ if target in blacklist:+ self.sf.debug(f"IP address {target} found in Abuse.ch Feodo Tracker.")+ return True+ elif targetType == "netblock":+ netblock = IPNetwork(target)+ for ip in blacklist:+ if IPAddress(ip) in netblock:+ self.sf.debug(f"IP address {ip} found within netblock/subnet {target} in Abuse.ch Feodo Tracker.")+ return True++ return False++ def retrieveFeodoTrackerBlacklist(self):+ blacklist = self.sf.cacheGet('abusech_feodo', 24)++ if blacklist is not None:+ return self.parseFeodoTrackerBlacklist(blacklist)++ res = self.sf.fetchUrl(+ "https://feodotracker.abuse.ch/downloads/ipblocklist.txt",+ timeout=self.opts['_fetchtimeout'],+ useragent=self.opts['_useragent'],+ )++ if res['code'] != "200":+ self.sf.error(f"Unexpected HTTP response code {res['code']} from Abuse.ch Abuse.ch Feodo Tracker.")+ self.errorState = True+ return None++ if res['content'] is None:+ self.sf.error("Received no content from Abuse.ch Feodo Tracker")+ self.errorState = True+ return None++ self.sf.cachePut("abusech_feodo", res['content'])++ return self.parseFeodoTrackerBlacklist(res['content'])++ def parseFeodoTrackerBlacklist(self, blacklist):+ """Parse plaintext blacklist++ Args:+ blacklist (str): plaintext blacklist from Abuse.ch Feodo Tracker++ Returns:+ list: list of blacklisted IP addresses+ """+ ips = list()++ if not blacklist:+ return ips++ for ip in blacklist.split('\n'):+ ip = ip.strip()+ if not ip:+ continue+ if ip.startswith('#'):+ continue+ if not self.sf.validIP(ip):+ continue+ ips.append(ip)++ return ips++ def querySslBlacklist(self, target, targetType):+ blacklist = self.retrieveSslBlacklist()++ if not blacklist:+ return False++ if targetType == "ip":+ if target in blacklist:+ self.sf.debug(f"IP address {target} found in Abuse.ch SSL Blacklist.")+ return True+ elif targetType == "netblock":+ netblock = IPNetwork(target)+ for ip in blacklist:+ if IPAddress(ip) in netblock:+ self.sf.debug(f"IP address {ip} found within netblock/subnet {target} in Abuse.ch SSL Blacklist.")+ return True++ return False++ def retrieveSslBlacklist(self):+ blacklist = self.sf.cacheGet('abusech_ssl', 24)++ if blacklist is not None:+ return self.parseSslBlacklist(blacklist)++ res = self.sf.fetchUrl(+ "https://sslbl.abuse.ch/blacklist/sslipblacklist.csv",+ timeout=self.opts['_fetchtimeout'],+ useragent=self.opts['_useragent'],+ )++ if res['code'] != "200":+ self.sf.error(f"Unexpected HTTP response code {res['code']} from Abuse.ch Abuse.ch Feodo Tracker.")+ self.errorState = True+ return None++ if res['content'] is None:+ self.sf.error("Received no content from Abuse.ch Feodo Tracker")+ self.errorState = True+ return None++ self.sf.cachePut("abusech_ssl", res['content'])++ return self.parseSslBlacklist(res['content'])++ def parseSslBlacklist(self, blacklist):+ """Parse plaintext blacklist++ Args:+ blacklist (str): CSV blacklist from Abuse.ch SSL Blacklist++ Returns:+ list: list of blacklisted IP addresses+ """+ ips = list()++ if not blacklist:+ return ips++ for line in blacklist.split('\n'):+ line = line.strip()+ if not line:+ continue+ if line.startswith('#'):+ continue+ csv = line.split(',')+ if len(csv) < 2:+ continue+ ip = csv[1]+ if not self.sf.validIP(ip):+ continue+ ips.append(ip)++ return ips++ def queryUrlHausBlacklist(self, target, targetType):+ blacklist = self.retrieveUrlHausBlacklist()++ if not blacklist:+ return False++ if targetType == "ip":+ if target in blacklist:+ self.sf.debug(f"IP address {target} found in Abuse.ch URL Haus Blacklist.")+ return True+ elif targetType == "netblock":+ netblock = IPNetwork(target)+ for ip in blacklist:+ if IPAddress(ip) in netblock:+ self.sf.debug(f"IP address {ip} found within netblock/subnet {target} in Abuse.ch URL Haus Blacklist.")+ return True+ elif targetType == "domain":+ if target.lower() in blacklist:+ self.sf.debug(f"Host name {target} found in Abuse.ch URL Haus Blacklist.")+ return True++ return False++ def retrieveUrlHausBlacklist(self):+ blacklist = self.sf.cacheGet('abusech_urlhaus', 24)++ if blacklist is not None:+ return self.parseUrlHausBlacklist(blacklist)++ res = self.sf.fetchUrl(+ "https://urlhaus.abuse.ch/downloads/csv_recent/",+ timeout=self.opts['_fetchtimeout'],+ useragent=self.opts['_useragent'],+ )++ if res['code'] != "200":+ self.sf.error(f"Unexpected HTTP response code {res['code']} from Abuse.ch URL Haus.")+ self.errorState = True+ return None++ if res['content'] is None:+ self.sf.error("Received no content from Abuse.ch URL Haus")+ self.errorState = True+ return None++ self.sf.cachePut("abusech_urlhaus", res['content'])++ return self.parseUrlHausBlacklist(res['content'])++ def parseUrlHausBlacklist(self, blacklist):+ """Parse plaintext blacklist++ Args:+ blacklist (str): plaintext blacklist from Abuse.ch URL Haus++ Returns:+ list: list of blacklisted hosts+ """+ hosts = list()++ if not blacklist:+ return hosts++ for line in blacklist.split('\n'):+ if not line:+ continue+ if line.startswith('#'):+ continue++ # Note: URL parsing and validation with sf.validHost() is too slow to use here+ url = line.strip().lower()+ if len(url.split("/")) < 3:+ continue+ host = url.split("/")[2].split(':')[0]+ if not host:+ continue+ if "." not in host:+ continue+ hosts.append(host)++ return hosts+def handleEvent(self, event):eventName = event.eventTypesrcModuleName = event.module@@ -230,57 +350,65 @@ class sfp_abusech(SpiderFootPlugin):self.sf.debug(f"Skipping {eventData}, already checked.")return+ if self.errorState:+ return+self.results[eventData] = True- if eventName == 'CO_HOSTED_SITE' and not self.opts.get('checkcohosts', False):- return- if eventName == 'AFFILIATE_IPADDR' \- and not self.opts.get('checkaffiliates', False):- return- if eventName == 'NETBLOCK_OWNER' and not self.opts.get('checknetblocks', False):- return- if eventName == 'NETBLOCK_MEMBER' and not self.opts.get('checksubnets', False):+ if eventName == 'IP_ADDRESS':+ targetType = 'ip'+ evtType = 'MALICIOUS_IPADDR'+ elif eventName == 'AFFILIATE_IPADDR':+ if not self.opts.get('checkaffiliates', False):+ return+ targetType = 'ip'+ evtType = 'MALICIOUS_AFFILIATE_IPADDR'+ elif eventName == 'NETBLOCK_OWNER':+ if not self.opts.get('checknetblocks', False):+ return+ targetType = 'netblock'+ evtType = 'MALICIOUS_NETBLOCK'+ elif eventName == 'NETBLOCK_MEMBER':+ if not self.opts.get('checksubnets', False):+ return+ targetType = 'netblock'+ evtType = 'MALICIOUS_SUBNET'+ elif eventName == "INTERNET_NAME":+ targetType = 'domain'+ evtType = "MALICIOUS_INTERNET_NAME"+ elif eventName == 'AFFILIATE_INTERNET_NAME':+ if not self.opts.get('checkaffiliates', False):+ return+ targetType = 'domain'+ evtType = 'MALICIOUS_AFFILIATE_INTERNET_NAME'+ elif eventName == 'CO_HOSTED_SITE':+ if not self.opts.get('checkcohosts', False):+ return+ targetType = 'domain'+ evtType = 'MALICIOUS_COHOST'+ else:return- for check in list(malchecks.keys()):- cid = malchecks[check]['id']-- if eventName in ['IP_ADDRESS', 'AFFILIATE_IPADDR']:- typeId = 'ip'- if eventName == 'IP_ADDRESS':- evtType = 'MALICIOUS_IPADDR'- else:- evtType = 'MALICIOUS_AFFILIATE_IPADDR'-- if eventName in ['BGP_AS_OWNER', 'BGP_AS_MEMBER']:- typeId = 'asn'- evtType = 'MALICIOUS_ASN'-- if eventName in ['INTERNET_NAME', 'CO_HOSTED_SITE',- 'AFFILIATE_INTERNET_NAME']:- typeId = 'domain'- if eventName == "INTERNET_NAME":- evtType = "MALICIOUS_INTERNET_NAME"- if eventName == 'AFFILIATE_INTERNET_NAME':- evtType = 'MALICIOUS_AFFILIATE_INTERNET_NAME'- if eventName == 'CO_HOSTED_SITE':- evtType = 'MALICIOUS_COHOST'-- if eventName == 'NETBLOCK_OWNER':- typeId = 'netblock'- evtType = 'MALICIOUS_NETBLOCK'- if eventName == 'NETBLOCK_MEMBER':- typeId = 'netblock'- evtType = 'MALICIOUS_SUBNET'-- url = self.lookupItem(cid, typeId, eventData)-- if self.checkForStop():- return+ if targetType in ['ip', 'netblock']:+ self.sf.debug(f"Checking maliciousness of {eventData} ({eventName}) with Abuse.ch Feodo Tracker")+ if self.queryFeodoTrackerBlacklist(eventData, targetType):+ url = "https://feodotracker.abuse.ch/downloads/ipblocklist.txt"+ text = f"Abuse.ch Feodo Tracker [{eventData}]\n<SFURL>{url}</SFURL>"+ evt = SpiderFootEvent(evtType, text, self.__name__, event)+ self.notifyListeners(evt)++ self.sf.debug(f"Checking maliciousness of {eventData} ({eventName}) with Abuse.ch SSL Blacklist")+ if self.querySslBlacklist(eventData, targetType):+ url = "https://sslbl.abuse.ch/blacklist/sslipblacklist.csv"+ text = f"Abuse.ch SSL Blacklist [{eventData}]\n<SFURL>{url}</SFURL>"+ evt = SpiderFootEvent(evtType, text, self.__name__, event)+ self.notifyListeners(evt)- # Notify other modules of what you've found- if url is not None:- text = f"{check} [{eventData}]\n<SFURL>{url}</SFURL>"+ if targetType in ['ip', 'domain']:+ self.sf.debug(f"Checking maliciousness of {eventData} ({eventName}) with Abuse.ch URL Haus")+ if self.queryUrlHausBlacklist(eventData, targetType):+ url = "https://urlhaus.abuse.ch/downloads/csv_recent/"+ text = f"Abuse.ch URL Haus Blacklist [{eventData}]\n<SFURL>{url}</SFURL>"evt = SpiderFootEvent(evtType, text, self.__name__, event)self.notifyListeners(evt)