SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.
Python attacksurfacecticybersecurityfootprintinghackinginformation-gatheringinformation-securityinfosecintelligence-gatheringosintosint-frameworkosint-reconnaissanceosint-toolpentestingpythonreconsecurity-toolsthreat-intelligencethreatintel
Modules for Wigle.net and searching for edits on wikipedia by a given IP address.
Steveeb7093df2657
diff --git a/modules/sfp_wigle.py b/modules/sfp_wigle.pynew file mode 100644index 00000000..be9ae9b8--- /dev/null+++ b/modules/sfp_wigle.py@@ -0,0 +1,152 @@+#-------------------------------------------------------------------------------+# Name: sfp_wigle+# Purpose: Query wigle.net to identify nearby WiFi access points.+#+# Author: Steve Micallef <[email protected]>+#+# Created: 10/09/2017+# Copyright: (c) Steve Micallef+# Licence: GPL+#-------------------------------------------------------------------------------++import sys+import json+import datetime+import urllib+from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent++class sfp_wigle(SpiderFootPlugin):+ """Wigle.net:Footprint,Investigate,Passive:Secondary Networks::Query wigle.net to identify nearby WiFi access points."""+++ # Default options+ opts = {+ "api_key_encoded": "",+ "days_limit": "365",+ "variance": "0.01"+ }++ # Option descriptions+ optdescs = {+ "api_key_encoded": "Your Base64-encoded API name/token pair, as provided on your Wigle.net account page.",+ "days_limit": "Maximum age of data to be considered valid.",+ "variance": "How tightly to bound queries against the latitude/longitude box extracted from idenified addresses. This value must be between 0.001 and 0.2."++ }++ # Be sure to completely clear any class variables in setup()+ # or you run the risk of data persisting between scan runs.++ results = dict()++ def setup(self, sfc, userOpts=dict()):+ self.sf = sfc+ self.results = dict()++ # Clear / reset any other class member variables here+ # or you risk them persisting between threads.++ for opt in userOpts.keys():+ self.opts[opt] = userOpts[opt]++ # What events is this module interested in for input+ def watchedEvents(self):+ return ["PHYSICAL_ADDRESS"]++ # What events this module produces+ def producedEvents(self):+ return ["WIFI_ACCESS_POINT"]++ def getcoords(self, qry):+ url = "https://api.wigle.net/api/v2/network/geocode?" + \+ urllib.urlencode({'addresscode': unicode.encode(qry, 'utf-8', errors='replace')})+ hdrs = {+ "Accept": "application/json",+ "Authorization": "Basic " + self.opts['api_key_encoded']+ }++ res = self.sf.fetchUrl(url, timeout=30,+ useragent="SpiderFoot", headers=hdrs)+ if res['code'] == "404":+ return None+ if "too many queries" in res['content']:+ self.sf.error("Wigle.net query limit reached for the day.", False)+ return None++ try:+ info = json.loads(res['content'])+ if len(info.get('results', [])) == 0:+ return None+ return info['results'][0]['boundingbox']+ except Exception as e:+ self.sf.error("Error processing JSON response from Wigle.net: " + str(e), False)+ return None++ def getnetworks(self, coords):+ url = "https://api.wigle.net/api/v2/network/search?onlymine=false&" + \+ "latrange1=" + str(coords[0]) + "&latrange2=" + str(coords[1]) + \+ "&longrange1=" + str(coords[2]) + "&longrange2=" + str(coords[3]) + \+ "&freenet=false&paynet=false&variance=" + self.opts['variance']++ if self.opts['days_limit'] != "0":+ dt = datetime.datetime.now() - datetime.timedelta(days=int(self.opts['days_limit']))+ date_calc = dt.strftime("%Y%m%d")+ url += "&lastupdt=" + date_calc++ hdrs = {+ "Accept": "application/json",+ "Authorization": "Basic " + self.opts['api_key_encoded']+ }++ res = self.sf.fetchUrl(url, timeout=30,+ useragent="SpiderFoot", headers=hdrs)+ if res['code'] == "404":+ return None+ if "too many queries" in res['content']:+ self.sf.error("Wigle.net query limit reached for the day.", False)+ return None++ ret = list()+ try:+ info = json.loads(res['content'])+ if len(info.get('results', [])) == 0:+ return None+ for r in info['results']:+ if None not in [r['ssid'], r['netid']]:+ ret.append(r['ssid'] + " (Net ID: " + r['netid'] + ")")+ return ret+ except Exception as e:+ self.sf.error("Error processing JSON response from Wigle.net: " + str(e), False)+ return None+++ # Handle events sent to this module+ def handleEvent(self, event):+ eventName = event.eventType+ srcModuleName = event.module+ eventData = event.data++ self.sf.debug("Received event, " + eventName + ", from " + srcModuleName)++ # Don't look up stuff twice+ if eventData in self.results:+ self.sf.debug("Skipping " + eventData + " as already mapped.")+ return None+ else:+ self.results[eventData] = True++ coords = self.getcoords(eventData)+ if not coords:+ self.sf.error("Couldn't get coordinates for address from Wigle.net.", False)+ return None++ nets = self.getnetworks(coords)+ if not nets:+ self.sf.error("Couldn't get networks for coordinates from Wigle.net.", False)+ return None++ for n in nets:+ e = SpiderFootEvent("WIFI_ACCESS_POINT", n, self.__name__, event)+ self.notifyListeners(e)++# End of sfp_wigle classdiff --git a/modules/sfp_wikipediaedits.py b/modules/sfp_wikipediaedits.pynew file mode 100644index 00000000..8ff771ca--- /dev/null+++ b/modules/sfp_wikipediaedits.py@@ -0,0 +1,110 @@+#-------------------------------------------------------------------------------+# Name: sfp_wikipediaedits+# Purpose: Identify edits to Wikipedia articles made from a given IP address+# or username.+#+# Author: Steve Micallef <[email protected]>+#+# Created: 10/09/2017+# Copyright: (c) Steve Micallef+# Licence: GPL+#-------------------------------------------------------------------------------++import sys+import json+import datetime+import urllib2+import re+from HTMLParser import HTMLParser+from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent++class sfp_wikipediaedits(SpiderFootPlugin):+ """Wikipedia Edits:Footprint,Investigate,Passive:Secondary Networks::Identify edits to Wikipedia articles made from a given IP address or username."""+++ # Default options+ opts = {+ "days_limit": "365"+ }++ # Option descriptions+ optdescs = {+ "days_limit": "Maximum age of data to be considered valid (0 = unlimited)."+ }++ # Be sure to completely clear any class variables in setup()+ # or you run the risk of data persisting between scan runs.++ results = dict()++ def setup(self, sfc, userOpts=dict()):+ self.sf = sfc+ self.results = dict()+ self.__dataSource__ = "Wikipedia"++ # Clear / reset any other class member variables here+ # or you risk them persisting between threads.++ for opt in userOpts.keys():+ self.opts[opt] = userOpts[opt]++ # What events is this module interested in for input+ def watchedEvents(self):+ return ["IP_ADDRESS", "USERNAME"]++ # What events this module produces+ def producedEvents(self):+ return ["WIKIPEDIA_PAGE_EDIT"]++ def query(self, qry):+ url = "https://en.wikipedia.org/w/api.php?action=feedcontributions&user=" + qry+ if self.opts['days_limit'] != "0":+ dt = datetime.datetime.now() - datetime.timedelta(days=int(self.opts['days_limit']))+ y = dt.strftime("%Y")+ m = dt.strftime("%m")+ url += "&year=" + y + "&month=" + m+ res = self.sf.fetchUrl(url, timeout=self.opts['_fetchtimeout'],+ useragent="SpiderFoot")+ if res['code'] in [ "404", "403", "500" ]:+ return None++ links = list()+ try:+ parser = HTMLParser()+ for line in res['content'].split("\n"):+ matches = re.findall("<link>(.*?)</link>", line, re.IGNORECASE)+ for m in matches:+ if "Special:Contributions" in m:+ continue+ d = parser.unescape(m)+ links.append(d)+ return links+ except Exception as e:+ self.sf.error("Error processing response from Wikipedia: " + str(e), False)+ return None++ # Handle events sent to this module+ def handleEvent(self, event):+ eventName = event.eventType+ srcModuleName = event.module+ eventData = event.data++ self.sf.debug("Received event, " + eventName + ", from " + srcModuleName)++ # Don't look up stuff twice+ if eventData in self.results:+ self.sf.debug("Skipping " + eventData + " as already mapped.")+ return None+ else:+ self.results[eventData] = True++ data = self.query(eventData)+ if data == None:+ return None++ for l in data:+ e = SpiderFootEvent("WIKIPEDIA_PAGE_EDIT", l,+ self.__name__, event)+ self.notifyListeners(e)++# End of sfp_wikipediaedits classdiff --git a/sfdb.py b/sfdb.pyindex 25b2fd3f..1a32066b 100644--- a/sfdb.py+++ b/sfdb.py@@ -206,7 +206,9 @@ class SpiderFootDb:"INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('WEBSERVER_BANNER', 'Web Server', 0, 'DATA')","INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('WEBSERVER_HTTPHEADERS', 'HTTP Headers', 1, 'DATA')","INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('WEBSERVER_STRANGEHEADER', 'Non-Standard HTTP Header', 0, 'DATA')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('WEBSERVER_TECHNOLOGY', 'Web Technology', 0, 'DESCRIPTOR')"+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('WEBSERVER_TECHNOLOGY', 'Web Technology', 0, 'DESCRIPTOR')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('WIFI_ACCESS_POINT', 'WiFi Access Point Nearby', 0, 'ENTITY')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('WIKIPEDIA_PAGE_EDIT', 'Wikipedia Page Edit', 0, 'DESCRIPTOR')"]def __init__(self, opts):