Skip to content

SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.

Update sfp_flickr to extract URLs and domains

Brendan Colesb159a8fd9259
diff --git a/modules/sfp_flickr.py b/modules/sfp_flickr.py
index 3fdbefab..f29acf6b 100644
--- a/modules/sfp_flickr.py
+++ b/modules/sfp_flickr.py
@@ -1,8 +1,8 @@
# -*- coding: utf-8 -*-
# -------------------------------------------------------------------------------
-# Name: sfp_flickr
-# Purpose: SpiderFoot plug-in for retrieving email addresses belonging
-# to your target from Flickr.
+# Name: sfp_flickr
+# Purpose: Search Flickr API for domains, URLs and emails related to the
+# specified domain.
#
# Author: <[email protected]>
#
@@ -13,26 +13,28 @@
import json
import re
-
import time
+import urllib.request, urllib.parse, urllib.error
from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
class sfp_flickr(SpiderFootPlugin):
- """Flickr:Footprint,Investigate,Passive:Social Media::Look up e-mail addresses on Flickr."""
+ """Flickr:Footprint,Investigate,Passive:Social Media::Search Flickr for domains, URLs and emails related to the specified domain."""
# Default options
opts = {
- 'pause': 1, # number of seconds to pause between fetches
- 'per_page': 200, # max number of results per page
- 'maxpages': 20 # max number of pages to fetch
+ 'pause': 1,
+ 'per_page': 100,
+ 'maxpages': 20,
+ 'dns_resolve': True,
}
# Option descriptions
optdescs = {
'pause': "Number of seconds to pause between fetches.",
'per_page': "Maximum number of results per page.",
- 'maxpages': "Maximum number of pages of results to fetch."
+ 'maxpages': "Maximum number of pages of results to fetch.",
+ 'dns_resolve': "DNS resolve each identified domain.",
}
results = None
@@ -46,11 +48,11 @@ class sfp_flickr(SpiderFootPlugin):
# What events is this module interested in for input
def watchedEvents(self):
- return ['INTERNET_NAME', "DOMAIN_NAME"]
+ return ["DOMAIN_NAME"]
# What events this module produces
def producedEvents(self):
- return ["EMAILADDR"]
+ return ["EMAILADDR", "INTERNET_NAME", "DOMAIN_NAME", "LINKED_URL_INTERNAL"]
# Retrieve API key
def retrieveApiKey(self):
@@ -68,15 +70,39 @@ class sfp_flickr(SpiderFootPlugin):
# Query the REST API
def query(self, qry, api_key, page=1, per_page=200):
- output = "json"
- url = "https://api.flickr.com/services/rest?"
- url += "sort=relevance&parse_tags=1&content_type=7&extras=description,owner_name,path_alias,realname&"
- url += "hermes=1&hermesClient=1&reqId=&nojsoncallback=1&viewerNSID=&method=flickr.photos.search&csrf=&lang=en-US&"
- url += "per_page=" + str(per_page) + "&page=" + str(page) + "&text=" + qry + "&api_key=" + api_key + "&format=" + output
-
- res = self.sf.fetchUrl(url, timeout=self.opts['_fetchtimeout'], useragent=self.opts['_useragent'])
+ params = {
+ "sort": "relevance",
+ "parse_tags": "1",
+ "content_type": "7",
+ "extras": "description,owner_name,path_alias,realname",
+ "hermes": "1",
+ "hermesClient": "1",
+ "reqId": "",
+ "nojsoncallback": "1",
+ "viewerNSID": "",
+ "method": "flickr.photos.search",
+ "csrf": "",
+ "lang": "en-US",
+ "per_page": str(per_page),
+ "page": str(page),
+ "text": qry.encode('raw_unicode_escape').decode("ascii", errors='replace'),
+ "api_key": api_key,
+ "format": "json"
+ }
+
+ res = self.sf.fetchUrl("https://api.flickr.com/services/rest?" + urllib.parse.urlencode(params),
+ useragent=self.opts['_useragent'],
+ timeout=self.opts['_fetchtimeout'])
+
+ time.sleep(self.opts['pause'])
+
+ try:
+ data = json.loads(res['content'])
+ except BaseException as e:
+ self.sf.debug("Error processing JSON response: " + str(e))
+ return None
- return res['content']
+ return data
# Handle events sent to this module
def handleEvent(self, event):
@@ -86,11 +112,15 @@ class sfp_flickr(SpiderFootPlugin):
if eventData in self.results:
return None
- else:
- self.results[eventData] = True
+
+ self.results[eventData] = True
self.sf.debug("Received event, " + eventName + ", from " + srcModuleName)
+ if srcModuleName == 'sfp_flickr':
+ self.sf.debug("Ignoring " + eventData + ", from self.")
+ return None
+
# Retrieve API key
api_key = self.retrieveApiKey()
@@ -100,7 +130,8 @@ class sfp_flickr(SpiderFootPlugin):
self.sf.debug("Retrieved API key: " + api_key)
- # Get e-mail addresses for this domain
+ # Query API for event data
+ hosts = list()
page = 1
pages = self.opts['maxpages']
per_page = self.opts['per_page']
@@ -108,59 +139,92 @@ class sfp_flickr(SpiderFootPlugin):
if self.checkForStop():
return None
- res = self.query("@"+eventData, api_key, page=page, per_page=per_page)
-
- if res is None:
+ if self.errorState:
return None
- # Parse response content as JSON
- try:
- data = json.loads(res)
- except Exception as e:
- self.sf.debug("Error processing JSON response.")
+ data = self.query(eventData, api_key, page=page, per_page=per_page)
+
+ if data is None:
return None
# Check the response is ok
- if not data['stat'] == "ok":
+ if data.get('stat') != "ok":
self.sf.debug("Error retrieving search results.")
return None
- if 'photos' not in data:
- return None
+ photos = data.get('photos')
- if 'pages' not in data['photos']:
+ if not photos:
return None
# Calculate number of pages to retrieve
- result_pages = int(data['photos']['pages'])
+ result_pages = int(photos.get('pages'))
if result_pages < pages:
pages = result_pages
- if 'max_allowed_pages' in data['photos']:
- allowed_pages = int(data['photos']['max_allowed_pages'])
+ if 'max_allowed_pages' in photos:
+ allowed_pages = int(photos.get('max_allowed_pages'))
if pages > allowed_pages:
pages = allowed_pages
self.sf.info("Parsing page " + str(page) + " of " + str(pages))
- # Extract emails
- for photo in data['photos']['photo']:
- emails = self.sf.parseEmails(str(bytes(photo).decode('unicode-escape')))
+ # Extract data
+ for photo in photos.get('photo'):
+ emails = self.sf.parseEmails(str(photo))
for email in emails:
- # Skip unrelated emails
- mailDom = email.lower().split('@')[1]
- if not self.getTarget().matches(mailDom, includeChildren=True, includeParents=True):
- self.sf.debug("Skipped address: " + email)
+ if email in self.results:
+ continue
+
+ mail_domain = email.lower().split('@')[1]
+
+ if not self.getTarget().matches(mail_domain, includeChildren=True, includeParents=True):
+ self.sf.debug("Skipped unrelated address: " + email)
continue
- if email not in self.results:
- self.sf.info("Found e-mail address: " + email)
- evt = SpiderFootEvent("EMAILADDR", email, self.__name__, event)
- self.notifyListeners(evt)
- self.results[email] = True
+ self.sf.info("Found e-mail address: " + email)
+ evt = SpiderFootEvent("EMAILADDR", email, self.__name__, event)
+ self.notifyListeners(evt)
+ self.results[email] = True
+
+ links = self.sf.extractUrls(str(photo))
+ for link in links:
+ if link in self.results:
+ continue
+
+ host = self.sf.urlFQDN(link)
+
+ if not self.getTarget().matches(host, includeChildren=True, includeParents=True):
+ self.sf.debug("Skipped unrelated link: " + link)
+ continue
+
+ hosts.append(host)
+
+ self.sf.debug("Found a URL: " + link)
+ evt = SpiderFootEvent('LINKED_URL_INTERNAL', link, self.__name__, event)
+ self.notifyListeners(evt)
+ self.results[link] = True
page += 1
- time.sleep(self.opts['pause'])
+
+ for host in set(hosts):
+ if self.checkForStop():
+ return None
+
+ if self.errorState:
+ return None
+
+ if self.opts['dns_resolve'] and not self.sf.resolveHost(host):
+ self.sf.debug("Host " + host + " could not be resolved")
+ evt = SpiderFootEvent("INTERNET_NAME_UNRESOLVED", host, self.__name__, event)
+ self.notifyListeners(evt)
+ continue
+
+ evt = SpiderFootEvent("INTERNET_NAME", host, self.__name__, event)
+ self.notifyListeners(evt)
+ if self.sf.isDomain(host, self.opts["_internettlds"]):
+ evt = SpiderFootEvent("DOMAIN_NAME", host, self.__name__, event)
+ self.notifyListeners(evt)
# End of sfp_flickr class