SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.
Python attacksurfacecticybersecurityfootprintinghackinginformation-gatheringinformation-securityinfosecintelligence-gatheringosintosint-frameworkosint-reconnaissanceosint-toolpentestingpythonreconsecurity-toolsthreat-intelligencethreatintel
New modules - Fraudguard.io, IPinfo.io, blockchain.info.
Steve05313a53ce7c
diff --git a/modules/sfp_bitcoin.py b/modules/sfp_bitcoin.pyindex 214b58be..4ab1ee47 100644--- a/modules/sfp_bitcoin.py+++ b/modules/sfp_bitcoin.py@@ -74,7 +74,7 @@ class sfp_bitcoin(SpiderFootPlugin):self.sf.debug("Received event, " + eventName + ", from " + srcModuleName)# thanks to https://stackoverflow.com/questions/21683680/regex-to-match-bitcoin-addresses- matches = re.findall("[\s:=]([13][a-km-zA-HJ-NP-Z1-9]{25,34})", eventData)+ matches = re.findall("[\s:=\>]([13][a-km-zA-HJ-NP-Z1-9]{25,34})", eventData)for m in matches:self.sf.debug("Bitcoin potential match: " + m)if self.check_bc(m):diff --git a/modules/sfp_blockchain.py b/modules/sfp_blockchain.pynew file mode 100644index 00000000..28186927--- /dev/null+++ b/modules/sfp_blockchain.py@@ -0,0 +1,75 @@+# -*- coding: utf-8 -*-+# -------------------------------------------------------------------------------+# Name: sfp_blockchain+# Purpose: SpiderFoot plug-in to look up a bitcoin wallet's balance by+# querying blockchain.info.+#+# Author: Steve Micallef <[email protected]>+#+# Created: 18/06/2017+# Copyright: (c) Steve Micallef 2017+# Licence: GPL+# -------------------------------------------------------------------------------++import json+from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+++class sfp_blockchain(SpiderFootPlugin):+ """Blockchain:Footprint,Investigate,Passive:Social::Queries blockchain.info to find the balance of identified bitcoin wallet addresses."""++ # Default options+ opts = {}+ results = dict()++ def setup(self, sfc, userOpts=dict()):+ self.sf = sfc+ self.results = dict()++ for opt in userOpts.keys():+ self.opts[opt] = userOpts[opt]++ # What events is this module interested in for input+ def watchedEvents(self):+ return ['BITCOIN_ADDRESS']++ # What events this module produces+ # This is to support the end user in selecting modules based on events+ # produced.+ def producedEvents(self):+ return ["BITCOIN_BALANCE"]++ # Handle events sent to this module+ def handleEvent(self, event):+ eventName = event.eventType+ srcModuleName = event.module+ eventData = event.data++ self.sf.debug("Received event, " + eventName + ", from " + srcModuleName)++ # Don't look up stuff twice+ if eventData in self.results:+ self.sf.debug("Skipping " + eventData + " as already mapped.")+ return None+ else:+ self.results[eventData] = True++ # Wallet balance+ res = self.sf.fetchUrl("https://blockchain.info/balance?active=" + eventData,+ timeout=self.opts['_fetchtimeout'], useragent=self.opts['_useragent'])+ if res['content'] is None:+ self.sf.info("No Blockchain info found for " + eventData)+ return None+ try:+ data = json.loads(res['content'])+ balance = float(data[eventData]['final_balance']) / 100000000+ except Exception as e:+ self.sf.debug("Error processing JSON response.")+ return None++ evt = SpiderFootEvent("BITCOIN_BALANCE", str(balance) + " BTC", self.__name__, event)+ self.notifyListeners(evt)++ return None++# End of sfp_blockchain classdiff --git a/modules/sfp_fraudguard.py b/modules/sfp_fraudguard.pynew file mode 100644index 00000000..0bf2b3aa--- /dev/null+++ b/modules/sfp_fraudguard.py@@ -0,0 +1,150 @@+# -*- coding: utf-8 -*-+# -------------------------------------------------------------------------------+# Name: sfp_fraudguard+# Purpose: Query fraudguard.io using their API+#+# Author: Steve Micallef+#+# Created: 18/06/2017+# Copyright: (c) Steve Micallef 2017+# Licence: GPL+# -------------------------------------------------------------------------------++import json+import base64+from datetime import datetime+import time+from netaddr import IPNetwork+from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent++class sfp_fraudguard(SpiderFootPlugin):+ """Fraudguard:Investigate,Passive:Search Engines:apikey:Obtain threat information from Fraudguard.io"""++ # Default options+ opts = {+ "fraudguard_api_key_account": "",+ "fraudguard_api_key_password": "",+ "age_limit_days": 90+ }++ # Option descriptions+ optdescs = {+ "fraudguard_api_key_account": "Your Fraudguard.io API username",+ "fraudguard_api_key_password": "Your Fraudguard.io API password",+ "age_limit_days": "Ignore any records older than this many days. 0 = unlimited."+ }++ # Be sure to completely clear any class variables in setup()+ # or you run the risk of data persisting between scan runs.++ results = dict()+ errorState = False++ def setup(self, sfc, userOpts=dict()):+ self.sf = sfc+ self.results = dict()++ # Clear / reset any other class member variables here+ # or you risk them persisting between threads.++ for opt in userOpts.keys():+ self.opts[opt] = userOpts[opt]++ # What events is this module interested in for input+ def watchedEvents(self):+ return ["IP_ADDRESS", "NETBLOCK_OWNER"]++ # What events this module produces+ def producedEvents(self):+ return [ "GEOINFO", "MALICIOUS_IPADDR", "MALICIOUS_NETBLOCK" ]++ def query(self, qry):+ fraudguard_url = "https://api.fraudguard.io/ip/" + qry+ headers = {+ 'Authorization': "Basic " + base64.b64encode(self.opts['fraudguard_api_key_account'] + ":" + self.opts['fraudguard_api_key_password'])+ }++ print str(headers)+ res = self.sf.fetchUrl(fraudguard_url , timeout=self.opts['_fetchtimeout'],+ useragent="SpiderFoot", headers=headers)++ if res['code'] in [ "400", "429", "500", "403" ]:+ self.sf.error("Fraudguard.io API key seems to have been rejected or you have exceeded usage limits for the month.", False)+ self.errorState = True+ return None++ if res['content'] is None:+ self.sf.info("No Fraudguard.io info found for " + qry)+ return None++ self.sf.debug(str(res['content']))++ try:+ info = json.loads(res['content'])+ except Exception as e:+ self.sf.error("Error processing JSON response from Fraudguard.io.", False)+ return None++ #print str(info)+ return info+++ # Handle events sent to this module+ def handleEvent(self, event):+ eventName = event.eventType+ srcModuleName = event.module+ eventData = event.data++ if self.errorState:+ return None++ self.sf.debug("Received event, " + eventName + ", from " + srcModuleName)++ if self.opts['fraudguard_api_key_account'] == "" or self.opts['fraudguard_api_key_password'] == "":+ self.sf.error("You enabled sfp_fraudguard but did not set an API username/password!", False)+ self.errorState = True+ return None++ # Don't look up stuff twice+ if eventData in self.results:+ self.sf.debug("Skipping " + eventData + " as already mapped.")+ return None+ else:+ self.results[eventData] = True++ qrylist = list()+ rtype = ""+ if eventName.startswith("NETBLOCK_"):+ rtype = "NETBLOCK"+ for ipaddr in IPNetwork(eventData):+ qrylist.append(str(ipaddr))+ self.results[str(ipaddr)] = True+ else:+ rtype = "IPADDR"+ qrylist.append(eventData)++ for addr in qrylist:+ if self.checkForStop():+ return None++ rec = self.query(addr)+ if rec is not None:+ self.sf.debug("Found results in Fraudguard.io")+ # 2016-12-24T07:25:35+00:00'+ created_dt = datetime.strptime(rec.get('discover_date'), '%Y-%m-%d %H:%M:%S')+ created_ts = int(time.mktime(created_dt.timetuple()))+ age_limit_ts = int(time.time()) - (86400 * self.opts['age_limit_days'])+ if self.opts['age_limit_days'] > 0 and created_ts < age_limit_ts:+ self.sf.debug("Record found but too old, skipping.")+ continue+ if "unknown" not in [rec['country'], rec['state'], rec['city']]:+ dat = rec['country'] + ", " + rec['state'] + ", " + rec['city']+ e = SpiderFootEvent("GEOINFO", dat, self.__name__, event)+ self.notifyListeners(e)++ if rec.get('threat') != "unknown":+ dat = rec['threat'] + " (risk level: " + rec['risk_level'] + ")"+ e = SpiderFootEvent("MALICIOUS_" + rtype, dat, self.__name__, event)+ self.notifyListeners(e)++# End of sfp_fraudguard classdiff --git a/modules/sfp_freegeoip.py b/modules/sfp_freegeoip.pyindex 64d729b8..e2f3c27c 100644--- a/modules/sfp_freegeoip.py+++ b/modules/sfp_freegeoip.py@@ -1,8 +1,8 @@# -*- coding: utf-8 -*-# --------------------------------------------------------------------------------# Name: sfp_geoip+# Name: sfp_freegeoip# Purpose: SpiderFoot plug-in to identify the Geo-location of IP addresses-# identified by other modules.+# identified by other modules using freegeoip.net.## Author: Steve Micallef <[email protected]>#@@ -15,8 +15,8 @@ import jsonfrom sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent-class sfp_geoip(SpiderFootPlugin):- """GeoIP:Footprint,Investigate,Passive:Networking::Identifies the physical location of IP addresses identified."""+class sfp_freegeoip(SpiderFootPlugin):+ """FreeGeoIP:Footprint,Investigate,Passive:Networking::Identifies the physical location of IP addresses identified using freegeoip.net."""# Default optionsopts = {}@@ -54,7 +54,7 @@ class sfp_geoip(SpiderFootPlugin):else:self.results[eventData] = True- res = self.sf.fetchUrl("http://freegeoip.net/json/" + eventData,+ res = self.sf.fetchUrl("https://freegeoip.net/json/" + eventData,timeout=self.opts['_fetchtimeout'], useragent=self.opts['_useragent'])if res['content'] is None:self.sf.info("No GeoIP info found for " + eventData)@@ -72,4 +72,4 @@ class sfp_geoip(SpiderFootPlugin):return None-# End of sfp_geoip class+# End of sfp_freegeoip classdiff --git a/modules/sfp_ipinfo.py b/modules/sfp_ipinfo.pynew file mode 100644index 00000000..35c37a80--- /dev/null+++ b/modules/sfp_ipinfo.py@@ -0,0 +1,96 @@+# -*- coding: utf-8 -*-+# -------------------------------------------------------------------------------+# Name: sfp_ipinfo+# Purpose: SpiderFoot plug-in to identify the Geo-location of IP addresses+# identified by other modules using ipinfo.io.+#+# Author: Steve Micallef <[email protected]>+#+# Created: 17/06/2017+# Copyright: (c) Steve Micallef 2017+# Licence: GPL+# -------------------------------------------------------------------------------++import json+from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+++class sfp_ipinfo(SpiderFootPlugin):+ """IPInfo.io:Footprint,Investigate,Passive:Networking::Identifies the physical location of IP addresses identified using ipinfo.io."""++ # Default options+ opts = {+ "api_key": ""+ }+ optdescs = {+ "api_key": "Your ipinfo.io access token."+ }+ results = dict()+ errorState = False++ def setup(self, sfc, userOpts=dict()):+ self.sf = sfc+ self.results = dict()+ self.errorState = False++ for opt in userOpts.keys():+ self.opts[opt] = userOpts[opt]++ # What events is this module interested in for input+ def watchedEvents(self):+ return ['IP_ADDRESS']++ # What events this module produces+ # This is to support the end user in selecting modules based on events+ # produced.+ def producedEvents(self):+ return ["GEOINFO"]++ # Handle events sent to this module+ def handleEvent(self, event):+ eventName = event.eventType+ srcModuleName = event.module+ eventData = event.data++ if self.errorState:+ return None++ self.sf.debug("Received event, " + eventName + ", from " + srcModuleName)++ if self.opts['api_key'] == "":+ self.sf.error("You enabled sfp_ipinfo but did not set an API key!", False)+ self.errorState = True+ return None++ # Don't look up stuff twice+ if eventData in self.results:+ self.sf.debug("Skipping " + eventData + " as already mapped.")+ return None+ else:+ self.results[eventData] = True++ res = self.sf.fetchUrl("https://ipinfo.io/" + eventData + "/json?token=" + self.opts['api_key'],+ timeout=self.opts['_fetchtimeout'], useragent=self.opts['_useragent'])++ if res['code'] == "429":+ self.sf.error("You are being rate-limited by ipinfo.io.", False)+ return++ if res['content'] is None:+ self.sf.info("No GeoIP info found for " + eventData)+ try:+ hostip = json.loads(res['content'])+ except Exception as e:+ self.sf.debug("Error processing JSON response.")+ return None++ self.sf.info("Found GeoIP for " + eventData + ": " + hostip['country'])+ countrycity = hostip['country'] + ", " + hostip.get('region', "Unknown region") + \+ ", " + hostip.get('city', "Unknown city")++ evt = SpiderFootEvent("GEOINFO", countrycity, self.__name__, event)+ self.notifyListeners(evt)++ return None++# End of sfp_ipinfo classdiff --git a/sfdb.py b/sfdb.pyindex a618c392..0ed8c2c2 100644--- a/sfdb.py+++ b/sfdb.py@@ -100,6 +100,7 @@ class SpiderFootDb:"INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('AMAZON_S3_BUCKET', 'Amazon S3 Bucket', 0, 'ENTITY')","INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('BASE64_DATA', 'Base64-encoded Data', 1, 'DATA')","INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('BITCOIN_ADDRESS', 'Bitcoin Address', 0, 'ENTITY')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('BITCOIN_BALANCE', 'Bitcoin Balance', 0, 'DESCRIPTOR')","INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('BGP_AS_OWNER', 'BGP AS Ownership', 0, 'ENTITY')","INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('BGP_AS_MEMBER', 'BGP AS Membership', 0, 'ENTITY')","INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('BGP_AS_PEER', 'BGP AS Peer', 0, 'ENTITY')",