SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.
Python attacksurfacecticybersecurityfootprintinghackinginformation-gatheringinformation-securityinfosecintelligence-gatheringosintosint-frameworkosint-reconnaissanceosint-toolpentestingpythonreconsecurity-toolsthreat-intelligencethreatintel
sfp_cinsscore: rewrite and remove unused code
Brendan Colese6fb79fd4e79
diff --git a/modules/sfp_cinsscore.py b/modules/sfp_cinsscore.pyindex d8d333a2..d7b95267 100644--- a/modules/sfp_cinsscore.py+++ b/modules/sfp_cinsscore.py@@ -1,9 +1,9 @@# -*- coding: utf-8 -*-# --------------------------------------------------------------------------------# Name: sfp_cinsscore-# Purpose: Checks if an IP is malicious according to the CINS Army List.+# Name: sfp_cinsscore+# Purpose: Checks if an IP address is malicious according to the CINS Army List.#-# Author: [email protected]+# Author: [email protected]## Created: 13/05/2018# Copyright: (c) Steve Micallef, 2018@@ -11,23 +11,10 @@# -------------------------------------------------------------------------------from netaddr import IPAddress, IPNetwork-import re-from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent-malchecks = {- 'cinsscore.com List': {- 'id': '_cinsscore',- 'type': 'list',- 'checks': ['ip'],- 'url': 'http://cinsscore.com/list/ci-badguys.txt'- }-}--class sfp_cinsscore(SpiderFootPlugin):- """CINS Army List:Investigate,Passive:Reputation Systems::Check if a netblock or IP is malicious according to cinsscore.com's Army List."""-+ """CINS Army List:Investigate,Passive:Reputation Systems::Check if a netblock or IP address is malicious according to cinsscore.com's Army List."""# Default optionsopts = {@@ -39,168 +26,71 @@ class sfp_cinsscore(SpiderFootPlugin):# Option descriptionsoptdescs = {- 'checkaffiliates': "Apply checks to affiliates?",+ 'checkaffiliates': "Apply checks to affiliate IP addresses?",'cacheperiod': "Hours to cache list data before re-fetching.",'checknetblocks': "Report if any malicious IPs are found within owned netblocks?",'checksubnets': "Check if any malicious IPs are found within the same subnet of the target?"}- # Be sure to completely clear any class variables in setup()- # or you run the risk of data persisting between scan runs.-results = None+ errorState = Falsedef setup(self, sfc, userOpts=dict()):self.sf = sfcself.results = self.tempStorage()-- # Clear / reset any other class member variables here- # or you risk them persisting between threads.+ self.errorState = Falsefor opt in list(userOpts.keys()):self.opts[opt] = userOpts[opt]# What events is this module interested in for input- # * = be notified about all events.def watchedEvents(self):- return ["IP_ADDRESS", "NETBLOCK_MEMBER", "AFFILIATE_IPADDR",- "NETBLOCK_OWNER"]+ return ["IP_ADDRESS", "AFFILIATE_IPADDR",+ "NETBLOCK_MEMBER", "NETBLOCK_OWNER"]# What events this module produces- # This is to support the end user in selecting modules based on events- # produced.def producedEvents(self):return ["MALICIOUS_IPADDR", "MALICIOUS_AFFILIATE_IPADDR","MALICIOUS_SUBNET", "MALICIOUS_NETBLOCK"]- # Check the regexps to see whether the content indicates maliciousness- def contentMalicious(self, content, goodregex, badregex):- # First, check for the bad indicators- if len(badregex) > 0:- for rx in badregex:- if re.match(rx, content, re.IGNORECASE | re.DOTALL):- self.sf.debug("Found to be bad against bad regex: " + rx)- return True-- # Finally, check for good indicators- if len(goodregex) > 0:- for rx in goodregex:- if re.match(rx, content, re.IGNORECASE | re.DOTALL):- self.sf.debug("Found to be good againt good regex: " + rx)- return False+ def query(self, qry, targetType):+ cid = "_cinsscore"+ url = "http://cinsscore.com/list/ci-badguys.txt"- # If nothing was matched, reply None- self.sf.debug("Neither good nor bad, unknown.")- return None-- # Look up 'query' type sources- def resourceQuery(self, id, target, targetType):- self.sf.debug("Querying " + id + " for maliciousness of " + target)- for check in list(malchecks.keys()):- cid = malchecks[check]['id']- if id == cid and malchecks[check]['type'] == "query":- url = str(malchecks[check]['url'])- res = self.sf.fetchUrl(url.format(target), timeout=self.opts['_fetchtimeout'], useragent=self.opts['_useragent'])- if res['content'] is None:- self.sf.error("Unable to fetch " + url.format(target), False)- return None- if self.contentMalicious(res['content'],- malchecks[check]['goodregex'],- malchecks[check]['badregex']):- return url.format(target)+ data = dict()+ data["content"] = self.sf.cacheGet("sfmal_" + cid, self.opts.get('cacheperiod', 0))- return None+ if data["content"] is None:+ data = self.sf.fetchUrl(url, timeout=self.opts['_fetchtimeout'], useragent=self.opts['_useragent'])- # Look up 'list' type resources- def resourceList(self, id, target, targetType):- targetDom = ''- # Get the base domain if we're supplied a domain- if targetType == "domain":- targetDom = self.sf.hostDomain(target, self.opts['_internettlds'])- if not targetDom:+ if data["code"] != "200":+ self.sf.error("Unable to fetch %s" % url, False)+ self.errorState = Truereturn None- for check in list(malchecks.keys()):- cid = malchecks[check]['id']- if id == cid and malchecks[check]['type'] == "list":- data = dict()- url = malchecks[check]['url']- data['content'] = self.sf.cacheGet("sfmal_" + cid, self.opts.get('cacheperiod', 0))- if data['content'] is None:- data = self.sf.fetchUrl(url, timeout=self.opts['_fetchtimeout'], useragent=self.opts['_useragent'])- if data['content'] is None:- self.sf.error("Unable to fetch " + url, False)- return None- else:- self.sf.cachePut("sfmal_" + cid, data['content'])-- # If we're looking at netblocks- if targetType == "netblock":- iplist = list()- # Get the regex, replace {0} with an IP address matcher to- # build a list of IP.- # Cycle through each IP and check if it's in the netblock.- if 'regex' in malchecks[check]:- rx = malchecks[check]['regex'].replace("{0}",- "(\d+\.\d+\.\d+\.\d+)")- pat = re.compile(rx, re.IGNORECASE)- self.sf.debug("New regex for " + check + ": " + rx)- for line in data['content'].split('\n'):- grp = re.findall(pat, line)- if len(grp) > 0:- #self.sf.debug("Adding " + grp[0] + " to list.")- iplist.append(grp[0])- else:- iplist = data['content'].split('\n')-- for ip in iplist:- if len(ip) < 8 or ip.startswith("#"):- continue- ip = ip.strip()-- try:- if IPAddress(ip) in IPNetwork(target):- self.sf.debug(ip + " found within netblock/subnet " +- target + " in " + check)- return url- except Exception as e:- self.sf.debug("Error encountered parsing: " + str(e))- continue+ if data["content"] is None:+ self.sf.error("Unable to fetch %s" % url, False)+ self.errorState = True+ return None- return None+ self.sf.cachePut("sfmal_" + cid, data['content'])- # If we're looking at hostnames/domains/IPs- if 'regex' not in malchecks[check]:- for line in data['content'].split('\n'):- if line == target or (targetType == "domain" and line == targetDom):- self.sf.debug(target + "/" + targetDom + " found in " + check + " list.")- return url- else:- # Check for the domain and the hostname- try:- rxDom = str(malchecks[check]['regex']).format(targetDom)- rxTgt = str(malchecks[check]['regex']).format(target)- for line in data['content'].split('\n'):- if (targetType == "domain" and re.match(rxDom, line, re.IGNORECASE)) or \- re.match(rxTgt, line, re.IGNORECASE):- self.sf.debug(target + "/" + targetDom + " found in " + check + " list.")- return url- except BaseException as e:- self.sf.debug("Error encountered parsing 2: " + str(e))- continue+ for line in data["content"].split('\n'):+ ip = line.strip().lower()- return None+ if targetType == "netblock":+ try:+ if IPAddress(ip) in IPNetwork(qry):+ self.sf.debug("%s found within netblock/subnet %s in cinsscore.com list." % (ip, qry))+ return url+ except Exception as e:+ self.sf.debug("Error encountered parsing: %s" % e)+ continue- def lookupItem(self, resourceId, itemType, target):- for check in list(malchecks.keys()):- cid = malchecks[check]['id']- if cid == resourceId and itemType in malchecks[check]['checks']:- self.sf.debug("Checking maliciousness of " + target + " (" +- itemType + ") with: " + cid)- if malchecks[check]['type'] == "query":- return self.resourceQuery(cid, target, itemType)- if malchecks[check]['type'] == "list":- return self.resourceList(cid, target, itemType)+ if targetType == "ip":+ if qry.lower() == ip:+ self.sf.debug("%s found in cinsscore.com list." % qry)+ return urlreturn None@@ -215,61 +105,42 @@ class sfp_cinsscore(SpiderFootPlugin):if eventData in self.results:self.sf.debug("Skipping " + eventData + ", already checked.")return None- else:- self.results[eventData] = True- if eventName == 'CO_HOSTED_SITE' and not self.opts.get('checkcohosts', False):- return None- if eventName == 'AFFILIATE_IPADDR' \- and not self.opts.get('checkaffiliates', False):- return None- if eventName == 'NETBLOCK_OWNER' and not self.opts.get('checknetblocks', False):- return None- if eventName == 'NETBLOCK_MEMBER' and not self.opts.get('checksubnets', False):+ if self.errorState:return None- for check in list(malchecks.keys()):- cid = malchecks[check]['id']+ self.results[eventData] = True- if eventName in ['IP_ADDRESS', 'AFFILIATE_IPADDR']:- typeId = 'ip'- if eventName == 'IP_ADDRESS':- evtType = 'MALICIOUS_IPADDR'- else:- evtType = 'MALICIOUS_AFFILIATE_IPADDR'-- if eventName in ['BGP_AS_OWNER', 'BGP_AS_MEMBER']:- typeId = 'asn'- evtType = 'MALICIOUS_ASN'-- if eventName in ['INTERNET_NAME', 'CO_HOSTED_SITE',- 'AFFILIATE_INTERNET_NAME']:- typeId = 'domain'- if eventName == "INTERNET_NAME":- evtType = "MALICIOUS_INTERNET_NAME"- if eventName == 'AFFILIATE_INTERNET_NAME':- evtType = 'MALICIOUS_AFFILIATE_INTERNET_NAME'- if eventName == 'CO_HOSTED_SITE':- evtType = 'MALICIOUS_COHOST'-- if eventName == 'NETBLOCK_OWNER':- typeId = 'netblock'- evtType = 'MALICIOUS_NETBLOCK'- if eventName == 'NETBLOCK_MEMBER':- typeId = 'netblock'- evtType = 'MALICIOUS_SUBNET'+ if eventName == 'IP_ADDRESS':+ targetType = 'ip'+ evtType = 'MALICIOUS_IPADDR'+ elif eventName == 'AFFILIATE_IPADDR':+ if not self.opts.get('checkaffiliates', False):+ return None+ targetType = 'ip'+ evtType = 'MALICIOUS_AFFILIATE_IPADDR'+ elif eventName == 'NETBLOCK_OWNER':+ if not self.opts.get('checknetblocks', False):+ return None+ targetType = 'netblock'+ evtType = 'MALICIOUS_NETBLOCK'+ elif eventName == 'NETBLOCK_MEMBER':+ if not self.opts.get('checksubnets', False):+ return None+ targetType = 'netblock'+ evtType = 'MALICIOUS_SUBNET'+ else:+ return None- url = self.lookupItem(cid, typeId, eventData)+ self.sf.debug("Checking maliciousness of %s with cinsscore.com" % eventData)- if self.checkForStop():- return None+ url = self.query(eventData, targetType)- # Notify other modules of what you've found- if url is not None:- text = check + " [" + eventData + "]\n" + "<SFURL>" + url + "</SFURL>"- evt = SpiderFootEvent(evtType, text, self.__name__, event)- self.notifyListeners(evt)+ if not url:+ return None- return None+ text = "cinsscore.com [%s]\n<SFURL>%s</SFURL>" % (eventData, url)+ evt = SpiderFootEvent(evtType, text, self.__name__, event)+ self.notifyListeners(evt)# End of sfp_cinsscore class