SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.
Python attacksurfacecticybersecurityfootprintinghackinginformation-gatheringinformation-securityinfosecintelligence-gatheringosintosint-frameworkosint-reconnaissanceosint-toolpentestingpythonreconsecurity-toolsthreat-intelligencethreatintel
Update sfp_flickr to extract URLs and domains
Brendan Colesb159a8fd9259
diff --git a/modules/sfp_flickr.py b/modules/sfp_flickr.pyindex 3fdbefab..f29acf6b 100644--- a/modules/sfp_flickr.py+++ b/modules/sfp_flickr.py@@ -1,8 +1,8 @@# -*- coding: utf-8 -*-# --------------------------------------------------------------------------------# Name: sfp_flickr-# Purpose: SpiderFoot plug-in for retrieving email addresses belonging-# to your target from Flickr.+# Name: sfp_flickr+# Purpose: Search Flickr API for domains, URLs and emails related to the+# specified domain.## Author: <[email protected]>#@@ -13,26 +13,28 @@import jsonimport re-import time+import urllib.request, urllib.parse, urllib.errorfrom sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEventclass sfp_flickr(SpiderFootPlugin):- """Flickr:Footprint,Investigate,Passive:Social Media::Look up e-mail addresses on Flickr."""+ """Flickr:Footprint,Investigate,Passive:Social Media::Search Flickr for domains, URLs and emails related to the specified domain."""# Default optionsopts = {- 'pause': 1, # number of seconds to pause between fetches- 'per_page': 200, # max number of results per page- 'maxpages': 20 # max number of pages to fetch+ 'pause': 1,+ 'per_page': 100,+ 'maxpages': 20,+ 'dns_resolve': True,}# Option descriptionsoptdescs = {'pause': "Number of seconds to pause between fetches.",'per_page': "Maximum number of results per page.",- 'maxpages': "Maximum number of pages of results to fetch."+ 'maxpages': "Maximum number of pages of results to fetch.",+ 'dns_resolve': "DNS resolve each identified domain.",}results = None@@ -46,11 +48,11 @@ class sfp_flickr(SpiderFootPlugin):# What events is this module interested in for inputdef watchedEvents(self):- return ['INTERNET_NAME', "DOMAIN_NAME"]+ return ["DOMAIN_NAME"]# What events this module producesdef producedEvents(self):- return ["EMAILADDR"]+ return ["EMAILADDR", "INTERNET_NAME", "DOMAIN_NAME", "LINKED_URL_INTERNAL"]# Retrieve API keydef retrieveApiKey(self):@@ -68,15 +70,39 @@ class sfp_flickr(SpiderFootPlugin):# Query the REST APIdef query(self, qry, api_key, page=1, per_page=200):- output = "json"- url = "https://api.flickr.com/services/rest?"- url += "sort=relevance&parse_tags=1&content_type=7&extras=description,owner_name,path_alias,realname&"- url += "hermes=1&hermesClient=1&reqId=&nojsoncallback=1&viewerNSID=&method=flickr.photos.search&csrf=&lang=en-US&"- url += "per_page=" + str(per_page) + "&page=" + str(page) + "&text=" + qry + "&api_key=" + api_key + "&format=" + output-- res = self.sf.fetchUrl(url, timeout=self.opts['_fetchtimeout'], useragent=self.opts['_useragent'])+ params = {+ "sort": "relevance",+ "parse_tags": "1",+ "content_type": "7",+ "extras": "description,owner_name,path_alias,realname",+ "hermes": "1",+ "hermesClient": "1",+ "reqId": "",+ "nojsoncallback": "1",+ "viewerNSID": "",+ "method": "flickr.photos.search",+ "csrf": "",+ "lang": "en-US",+ "per_page": str(per_page),+ "page": str(page),+ "text": qry.encode('raw_unicode_escape').decode("ascii", errors='replace'),+ "api_key": api_key,+ "format": "json"+ }++ res = self.sf.fetchUrl("https://api.flickr.com/services/rest?" + urllib.parse.urlencode(params),+ useragent=self.opts['_useragent'],+ timeout=self.opts['_fetchtimeout'])++ time.sleep(self.opts['pause'])++ try:+ data = json.loads(res['content'])+ except BaseException as e:+ self.sf.debug("Error processing JSON response: " + str(e))+ return None- return res['content']+ return data# Handle events sent to this moduledef handleEvent(self, event):@@ -86,11 +112,15 @@ class sfp_flickr(SpiderFootPlugin):if eventData in self.results:return None- else:- self.results[eventData] = True++ self.results[eventData] = Trueself.sf.debug("Received event, " + eventName + ", from " + srcModuleName)+ if srcModuleName == 'sfp_flickr':+ self.sf.debug("Ignoring " + eventData + ", from self.")+ return None+# Retrieve API keyapi_key = self.retrieveApiKey()@@ -100,7 +130,8 @@ class sfp_flickr(SpiderFootPlugin):self.sf.debug("Retrieved API key: " + api_key)- # Get e-mail addresses for this domain+ # Query API for event data+ hosts = list()page = 1pages = self.opts['maxpages']per_page = self.opts['per_page']@@ -108,59 +139,92 @@ class sfp_flickr(SpiderFootPlugin):if self.checkForStop():return None- res = self.query("@"+eventData, api_key, page=page, per_page=per_page)-- if res is None:+ if self.errorState:return None- # Parse response content as JSON- try:- data = json.loads(res)- except Exception as e:- self.sf.debug("Error processing JSON response.")+ data = self.query(eventData, api_key, page=page, per_page=per_page)++ if data is None:return None# Check the response is ok- if not data['stat'] == "ok":+ if data.get('stat') != "ok":self.sf.debug("Error retrieving search results.")return None- if 'photos' not in data:- return None+ photos = data.get('photos')- if 'pages' not in data['photos']:+ if not photos:return None# Calculate number of pages to retrieve- result_pages = int(data['photos']['pages'])+ result_pages = int(photos.get('pages'))if result_pages < pages:pages = result_pages- if 'max_allowed_pages' in data['photos']:- allowed_pages = int(data['photos']['max_allowed_pages'])+ if 'max_allowed_pages' in photos:+ allowed_pages = int(photos.get('max_allowed_pages'))if pages > allowed_pages:pages = allowed_pagesself.sf.info("Parsing page " + str(page) + " of " + str(pages))- # Extract emails- for photo in data['photos']['photo']:- emails = self.sf.parseEmails(str(bytes(photo).decode('unicode-escape')))+ # Extract data+ for photo in photos.get('photo'):+ emails = self.sf.parseEmails(str(photo))for email in emails:- # Skip unrelated emails- mailDom = email.lower().split('@')[1]- if not self.getTarget().matches(mailDom, includeChildren=True, includeParents=True):- self.sf.debug("Skipped address: " + email)+ if email in self.results:+ continue++ mail_domain = email.lower().split('@')[1]++ if not self.getTarget().matches(mail_domain, includeChildren=True, includeParents=True):+ self.sf.debug("Skipped unrelated address: " + email)continue- if email not in self.results:- self.sf.info("Found e-mail address: " + email)- evt = SpiderFootEvent("EMAILADDR", email, self.__name__, event)- self.notifyListeners(evt)- self.results[email] = True+ self.sf.info("Found e-mail address: " + email)+ evt = SpiderFootEvent("EMAILADDR", email, self.__name__, event)+ self.notifyListeners(evt)+ self.results[email] = True++ links = self.sf.extractUrls(str(photo))+ for link in links:+ if link in self.results:+ continue++ host = self.sf.urlFQDN(link)++ if not self.getTarget().matches(host, includeChildren=True, includeParents=True):+ self.sf.debug("Skipped unrelated link: " + link)+ continue++ hosts.append(host)++ self.sf.debug("Found a URL: " + link)+ evt = SpiderFootEvent('LINKED_URL_INTERNAL', link, self.__name__, event)+ self.notifyListeners(evt)+ self.results[link] = Truepage += 1- time.sleep(self.opts['pause'])++ for host in set(hosts):+ if self.checkForStop():+ return None++ if self.errorState:+ return None++ if self.opts['dns_resolve'] and not self.sf.resolveHost(host):+ self.sf.debug("Host " + host + " could not be resolved")+ evt = SpiderFootEvent("INTERNET_NAME_UNRESOLVED", host, self.__name__, event)+ self.notifyListeners(evt)+ continue++ evt = SpiderFootEvent("INTERNET_NAME", host, self.__name__, event)+ self.notifyListeners(evt)+ if self.sf.isDomain(host, self.opts["_internettlds"]):+ evt = SpiderFootEvent("DOMAIN_NAME", host, self.__name__, event)+ self.notifyListeners(evt)# End of sfp_flickr class