SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.
Python attacksurfacecticybersecurityfootprintinghackinginformation-gatheringinformation-securityinfosecintelligence-gatheringosintosint-frameworkosint-reconnaissanceosint-toolpentestingpythonreconsecurity-toolsthreat-intelligencethreatintel
Add sfp_googlesafebrowsing module
Filip Aleksić3e6dcf867162
diff --git a/modules/sfp_googlesafebrowsing.py b/modules/sfp_googlesafebrowsing.pynew file mode 100644index 00000000..355d4ec1--- /dev/null+++ b/modules/sfp_googlesafebrowsing.py@@ -0,0 +1,220 @@+# -*- coding: utf-8 -*-+# -------------------------------------------------------------------------------+# Name: sfp_googlesafebrowsing+# Purpose: SpiderFoot plug-in to check if the URL is included on any of the+# Google Safe Browsing lists+#+# Author: Filip Aleksić <[email protected]>+#+# Created: 2020-08-18+# Copyright: (c) Filip Aleksić+# Licence: GPL+# -------------------------------------------------------------------------------++import json+from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent+++class sfp_googlesafebrowsing(SpiderFootPlugin):+ """GoogleSafeBrowsing:Investigate,Passive:Reputation Systems:apikey:Check if the URL is included on any of the Safe Browsing lists"""++ meta = {+ "name": "GoogleSafeBrowsing",+ "summary": "Check if the URL is included on any of the Safe Browsing lists.",+ "flags": ["slow", "apikey"],+ "useCases": ["Passive", "Investigate"],+ "categories": ["Reputation Systems"],+ "dataSource": {+ "website": "https://developers.google.com/safe-browsing/v4/lookup-api",+ "model": "FREE_AUTH_UNLIMITED",+ "references": [+ "https://developers.google.com/safe-browsing/v4/reference/rest"+ ],+ "apiKeyInstructions": [+ "Visit https://console.developers.google.com/",+ "Register a free Google account or sign in",+ "Create or select existing Google Developer Console project",+ "Go to the Cloud Console API Library https://console.cloud.google.com/apis/library",+ "From the projects list, select the project you want to use",+ "In the API Library select 'Safe Browsing APIs'",+ "On the API page, click ENABLE",+ "Navigate to the APIs & Services→Credentials panel in Cloud Console",+ "Select Create credentials, then select API key from the dropdown menu",+ "The API key created dialog box displays your newly created key",+ ],+ "favIcon": "https://www.gstatic.com/devrel-devsite/prod/v1241c04ebcb2127897d6c18221acbd64e7ed5c46e5217fd83dd808e592c47bf6/developers/images/favicon.png",+ "logo": "https://developers.google.com/safe-browsing/images/SafeBrowsing_Icon.png",+ "description": "The Safe Browsing APIs (v4) let your client applications check URLs "+ "against Google's constantly updated lists of unsafe web resources. "+ "Any URL found on a Safe Browsing list is considered unsafe.",+ },+ }++ opts = {"api_key": ""}++ optdescs = {+ "api_key": "Google Safe Browsing API key, it must be created and API enabled in the Google Developer Console. Without this you cannot obtain their feed.",+ }++ results = None+ errorState = False++ def setup(self, sfc, userOpts=dict()):+ self.sf = sfc+ self.results = self.tempStorage()++ for opt in list(userOpts.keys()):+ self.opts[opt] = userOpts[opt]++ # What events is this module interested in for input+ def watchedEvents(self):+ return [+ "INTERNET_NAME",+ "IP_ADDRESS",+ "AFFILIATE_INTERNET_NAME",+ "AFFILIATE_IPADDR",+ "CO_HOSTED_SITE",+ ]++ # What events this module produces+ # This is to support the end user in selecting modules based on events+ # produced.+ def producedEvents(self):+ return [+ "MALICIOUS_IPADDR",+ "MALICIOUS_INTERNET_NAME",+ "MALICIOUS_AFFILIATE_IPADDR",+ "MALICIOUS_AFFILIATE_INTERNET_NAME",+ "MALICIOUS_COHOST",+ "RAW_RIR_DATA",+ ]++ def query(self, qry):++ headers = {"Content-Type": "application/json"}+ url = (+ "https://safebrowsing.googleapis.com/v4/threatMatches"+ f":find?key={self.opts['api_key']}"+ )+ payload = {+ "client": {"clientId": "SpiderFoot", "clientVersion": "3.2",},+ "threatInfo": {+ "threatTypes": [+ "THREAT_TYPE_UNSPECIFIED",+ "MALWARE",+ "SOCIAL_ENGINEERING",+ "UNWANTED_SOFTWARE",+ "POTENTIALLY_HARMFUL_APPLICATION",+ ],+ "platformTypes": ["PLATFORM_TYPE_UNSPECIFIED", "ANY_PLATFORM"],+ "threatEntryTypes": [+ "THREAT_ENTRY_TYPE_UNSPECIFIED",+ "URL",+ "EXECUTABLE",+ ],+ "threatEntries": [+ {+ "url": qry.encode("raw_unicode_escape").decode(+ "ascii", errors="replace"+ )+ }+ ],+ },+ }+ res = self.sf.fetchUrl(+ url,+ timeout=self.opts["_fetchtimeout"],+ useragent=self.opts["_useragent"],+ headers=headers,+ postData=json.dumps(payload),+ )++ if res["code"] == "400":+ self.sf.error("Invalid request payload on Google Safe Browsing API", False)+ self.errorState = True+ return None++ if res["code"] == "429":+ self.sf.error("Reaching rate limit on Google Safe Browsing API", False)+ self.errorState = True+ return None++ if res["code"] == "403":+ self.sf.error(+ "Permission denied, invalid API key on Google Safe Browsing API", False+ )+ self.errorState = True+ return None++ if res["code"] in ["500", "503", "504"]:+ self.sf.error(+ "Google Safe Browsing API is having some troubles or unavailable.",+ False,+ )+ self.errorState = True+ return None++ try:+ info = json.loads(res["content"])+ if info == {}:+ self.sf.info("No Google Safe Browsing matches found for " + qry)+ return None++ except Exception as e:+ self.sf.error("Error processing JSON response from SHODAN.", False)+ return None++ return info++ def handleEvent(self, event):+ eventName = event.eventType+ srcModuleName = event.module+ eventData = event.data++ if self.errorState:+ return None++ self.sf.debug("Received event, %s, from %s" % (eventName, srcModuleName))++ if self.opts["api_key"] == "":+ self.sf.error("You enabled sfp_template but did not set an API key!", False)+ self.errorState = True+ return None++ # Don't look up stuff twice+ if eventData in self.results:+ self.sf.debug("Skipping " + eventData + " as already mapped.")+ return None+ else:+ self.results[eventData] = True++ evtType = ""+ if eventName in ["IP_ADDRESS", "AFFILIATE_IPADDR"]:+ if eventName == "IP_ADDRESS":+ evtType = "MALICIOUS_IPADDR"+ else:+ evtType = "MALICIOUS_AFFILIATE_IPADDR"++ if eventName in ["INTERNET_NAME", "CO_HOSTED_SITE", "AFFILIATE_INTERNET_NAME"]:+ if eventName == "INTERNET_NAME":+ evtType = "MALICIOUS_INTERNET_NAME"+ if eventName == "AFFILIATE_INTERNET_NAME":+ evtType = "MALICIOUS_AFFILIATE_INTERNET_NAME"+ if eventName == "CO_HOSTED_SITE":+ evtType = "MALICIOUS_COHOST"++ rec = self.query(eventData)++ if rec is None:+ return None++ evt = SpiderFootEvent("RAW_RIR_DATA", str(rec), self.__name__, pevent)+ self.notifyListeners(evt)++ evt = SpiderFootEvent(+ evtType, "GoogleSafeBrowsing [" + eventData + "]", self.__name__, event+ )+ self.notifyListeners(evt)+++# End of sfp_template class