SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.
Python attacksurfacecticybersecurityfootprintinghackinginformation-gatheringinformation-securityinfosecintelligence-gatheringosintosint-frameworkosint-reconnaissanceosint-toolpentestingpythonreconsecurity-toolsthreat-intelligencethreatintel
sfp_vxvault: Cleanup (#1291)
bcoleseed7e6928689
diff --git a/modules/sfp_vxvault.py b/modules/sfp_vxvault.pyindex 2e42124e..e95d4e20 100644--- a/modules/sfp_vxvault.py+++ b/modules/sfp_vxvault.py@@ -1,7 +1,7 @@# -*- coding: utf-8 -*-# -------------------------------------------------------------------------------# Name: sfp_vxvault-# Purpose: Checks if an ASN, IP or domain is malicious.+# Purpose: Check if a domain or IP address is malicious according to VXVault.net.## Author: [email protected]#@@ -10,27 +10,14 @@# Licence: GPL# --------------------------------------------------------------------------------import re--from netaddr import IPAddress, IPNetwork-from spiderfoot import SpiderFootEvent, SpiderFootPlugin-malchecks = {- 'VXVault Malicious URL List': {- 'id': '_vxvault',- 'checks': ['ip', 'domain'],- 'url': 'http://vxvault.net/URL_List.php',- 'regex': r'.*\/{0}/.*'- }-}-class sfp_vxvault(SpiderFootPlugin):meta = {'name': "VXVault.net",- 'summary': "Check if a domain or IP is malicious according to VXVault.net.",+ 'summary': "Check if a domain or IP address is malicious according to VXVault.net.",'flags': [],'useCases': ["Investigate", "Passive"],'categories': ["Reputation Systems"],@@ -42,7 +29,7 @@ class sfp_vxvault(SpiderFootPlugin):"https://github.com/InfectedPacket/VxVault"],'favIcon': "",- 'logo': "",+ 'logo': "http://vxvault.net/header.png",'description': "VxVault is a malware management program to automatically download and classify malware samples. ""VxVault downloads malware samples from links from online sources such as webpages or RSS feeds, ""downloads them and attempts to identify the malware using VirusTotal. "@@ -64,122 +51,104 @@ class sfp_vxvault(SpiderFootPlugin):'cacheperiod': "Hours to cache list data before re-fetching."}- # Be sure to completely clear any class variables in setup()- # or you run the risk of data persisting between scan runs.-results = None+ errorState = Falsedef setup(self, sfc, userOpts=dict()):self.sf = sfcself.results = self.tempStorage()-- # Clear / reset any other class member variables here- # or you risk them persisting between threads.+ self.errorState = Falsefor opt in list(userOpts.keys()):self.opts[opt] = userOpts[opt]# What events is this module interested in for input- # * = be notified about all events.def watchedEvents(self):- return ["INTERNET_NAME", "IP_ADDRESS",- "AFFILIATE_INTERNET_NAME", "AFFILIATE_IPADDR",- "CO_HOSTED_SITE"]+ return [+ "INTERNET_NAME",+ "IP_ADDRESS",+ "AFFILIATE_INTERNET_NAME",+ "AFFILIATE_IPADDR",+ "CO_HOSTED_SITE"+ ]# What events this module produces- # This is to support the end user in selecting modules based on events- # produced.def producedEvents(self):- return ["MALICIOUS_IPADDR", "MALICIOUS_INTERNET_NAME",- "MALICIOUS_AFFILIATE_IPADDR", "MALICIOUS_AFFILIATE_INTERNET_NAME",- "MALICIOUS_COHOST"]-- # Look up 'list' type resources- def resourceList(self, id, target, targetType):- targetDom = ''- # Get the base domain if we're supplied a domain- if targetType == "domain":- targetDom = self.sf.hostDomain(target, self.opts['_internettlds'])- if not targetDom:- return None-- for check in list(malchecks.keys()):- cid = malchecks[check]['id']- if id == cid:- data = dict()- url = malchecks[check]['url']- data['content'] = self.sf.cacheGet("sfmal_" + cid, self.opts.get('cacheperiod', 0))- if data['content'] is None:- data = self.sf.fetchUrl(url, timeout=self.opts['_fetchtimeout'], useragent=self.opts['_useragent'])- if data['content'] is None:- self.sf.error("Unable to fetch " + url)- return None-- self.sf.cachePut("sfmal_" + cid, data['content'])-- # If we're looking at netblocks- if targetType == "netblock":- iplist = list()- # Get the regex, replace {0} with an IP address matcher to- # build a list of IP.- # Cycle through each IP and check if it's in the netblock.- if 'regex' in malchecks[check]:- rx = malchecks[check]['regex'].replace("{0}", r"(\d+\.\d+\.\d+\.\d+)")- pat = re.compile(rx, re.IGNORECASE)- self.sf.debug("New regex for " + check + ": " + rx)- for line in data['content'].split('\n'):- grp = re.findall(pat, line)- if len(grp) > 0:- # self.sf.debug("Adding " + grp[0] + " to list.")- iplist.append(grp[0])- else:- iplist = data['content'].split('\n')-- for ip in iplist:- if len(ip) < 8 or ip.startswith("#"):- continue- ip = ip.strip()-- try:- if IPAddress(ip) in IPNetwork(target):- self.sf.debug(f"{ip} found within netblock/subnet {target} in {check}")- return url- except Exception as e:- self.sf.debug(f"Error encountered parsing: {e}")- continue-- return None-- # If we're looking at hostnames/domains/IPs- if 'regex' not in malchecks[check]:- for line in data['content'].split('\n'):- if line == target or (targetType == "domain" and line == targetDom):- self.sf.debug(target + "/" + targetDom + " found in " + check + " list.")- return url- else:- # Check for the domain and the hostname- try:- rxDom = str(malchecks[check]['regex']).format(targetDom)- rxTgt = str(malchecks[check]['regex']).format(target)- for line in data['content'].split('\n'):- if (targetType == "domain" and re.match(rxDom, line, re.IGNORECASE)) or \- re.match(rxTgt, line, re.IGNORECASE):- self.sf.debug(target + "/" + targetDom + " found in " + check + " list.")- return url- except Exception as e:- self.sf.debug("Error encountered parsing 2: " + str(e))- continue-- return None-- def lookupItem(self, resourceId, itemType, target):- for check in list(malchecks.keys()):- cid = malchecks[check]['id']- if cid == resourceId and itemType in malchecks[check]['checks']:- self.sf.debug("Checking maliciousness of " + target + " (" + itemType + ") with: " + cid)- return self.resourceList(cid, target, itemType)-- return None+ return [+ "MALICIOUS_IPADDR",+ "MALICIOUS_INTERNET_NAME",+ "MALICIOUS_AFFILIATE_IPADDR",+ "MALICIOUS_AFFILIATE_INTERNET_NAME",+ "MALICIOUS_COHOST"+ ]++ def queryBlacklist(self, target):+ blacklist = self.retrieveBlacklist()++ if target.lower() in blacklist:+ self.sf.debug(f"Host name {target} found in VXVault.net blacklist.")+ return True++ return False++ def retrieveBlacklist(self):+ blacklist = self.sf.cacheGet('vxvault', 24)++ if blacklist is not None:+ return self.parseBlacklist(blacklist)++ res = self.sf.fetchUrl(+ "http://vxvault.net/URL_List.php",+ timeout=10,+ useragent=self.opts['_useragent'],+ )++ if res['code'] != "200":+ self.sf.error(f"Unexpected HTTP response code {res['code']} from VXVault.net.")+ self.errorState = True+ return++ if res['content'] is None:+ self.sf.error("Received no content from VXVault.net")+ self.errorState = True+ return++ self.sf.cachePut("vxvault", res['content'])++ return self.parseBlacklist(res['content'])++ def parseBlacklist(self, blacklist):+ """Parse plaintext blacklist++ Args:+ blacklist (str): plaintext blacklist from VXVault.net++ Returns:+ list: list of blacklisted IP addresses and host names+ """+ hosts = list()++ if not blacklist:+ return hosts++ for line in blacklist.split('\n'):+ if not line:+ continue+ if not line.startswith('http'):+ continue++ # Note: URL parsing and validation with sf.validHost() is too slow to use here+ url = line.strip().lower()+ if not len(url.split("/")) >= 3:+ continue+ host = url.split("/")[2]+ if not host:+ continue+ if "." not in host:+ continue+ hosts.append(host)++ return hosts# Handle events sent to this moduledef handleEvent(self, event):@@ -193,47 +162,36 @@ class sfp_vxvault(SpiderFootPlugin):self.sf.debug(f"Skipping {eventData}, already checked.")return+ if self.errorState:+ return+self.results[eventData] = True- if eventName == 'CO_HOSTED_SITE' and not self.opts.get('checkcohosts', False):- return- if eventName == 'AFFILIATE_IPADDR' \- and not self.opts.get('checkaffiliates', False):+ if eventName == "IP_ADDRESS":+ evtType = 'MALICIOUS_IPADDR'+ elif eventName == "AFFILIATE_IPADDR":+ if not self.opts.get('checkaffiliates', False):+ return+ evtType = 'MALICIOUS_AFFILIATE_IPADDR'+ elif eventName == "INTERNET_NAME":+ evtType = "MALICIOUS_INTERNET_NAME"+ elif eventName == 'AFFILIATE_INTERNET_NAME':+ if not self.opts.get('checkaffiliates', False):+ return+ evtType = 'MALICIOUS_AFFILIATE_INTERNET_NAME'+ elif eventName == 'CO_HOSTED_SITE':+ if not self.opts.get('checkcohosts', False):+ return+ evtType = 'MALICIOUS_COHOST'+ else:return- for check in list(malchecks.keys()):- cid = malchecks[check]['id']-- if eventName in ['IP_ADDRESS', 'AFFILIATE_IPADDR']:- typeId = 'ip'- if eventName == 'IP_ADDRESS':- evtType = 'MALICIOUS_IPADDR'- else:- evtType = 'MALICIOUS_AFFILIATE_IPADDR'-- if eventName in ['BGP_AS_OWNER', 'BGP_AS_MEMBER']:- typeId = 'asn'- evtType = 'MALICIOUS_ASN'-- if eventName in ['INTERNET_NAME', 'CO_HOSTED_SITE',- 'AFFILIATE_INTERNET_NAME']:- typeId = 'domain'- if eventName == "INTERNET_NAME":- evtType = "MALICIOUS_INTERNET_NAME"- if eventName == 'AFFILIATE_INTERNET_NAME':- evtType = 'MALICIOUS_AFFILIATE_INTERNET_NAME'- if eventName == 'CO_HOSTED_SITE':- evtType = 'MALICIOUS_COHOST'-- url = self.lookupItem(cid, typeId, eventData)-- if self.checkForStop():- return+ self.sf.debug(f"Checking maliciousness of {eventData} ({eventName}) with VXVault.net")- # Notify other modules of what you've found- if url is not None:- text = f"{check} [{eventData}]\n<SFURL>{url}</SFURL>"- evt = SpiderFootEvent(evtType, text, self.__name__, event)- self.notifyListeners(evt)+ if self.queryBlacklist(eventData):+ url = "http://vxvault.net/URL_List.php"+ text = f"VXVault Malicious URL List [{eventData}]\n<SFURL>{url}</SFURL>"+ evt = SpiderFootEvent(evtType, text, self.__name__, event)+ self.notifyListeners(evt)# End of sfp_vxvault class