Skip to content

SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.

Modules for Wigle.net and searching for edits on wikipedia by a given IP address.

Steveeb7093df2657
diff --git a/modules/sfp_wigle.py b/modules/sfp_wigle.py
new file mode 100644
index 00000000..be9ae9b8
--- /dev/null
+++ b/modules/sfp_wigle.py
@@ -0,0 +1,152 @@
+#-------------------------------------------------------------------------------
+# Name: sfp_wigle
+# Purpose: Query wigle.net to identify nearby WiFi access points.
+#
+# Author: Steve Micallef <[email protected]>
+#
+# Created: 10/09/2017
+# Copyright: (c) Steve Micallef
+# Licence: GPL
+#-------------------------------------------------------------------------------
+
+import sys
+import json
+import datetime
+import urllib
+from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+
+class sfp_wigle(SpiderFootPlugin):
+ """Wigle.net:Footprint,Investigate,Passive:Secondary Networks::Query wigle.net to identify nearby WiFi access points."""
+
+
+ # Default options
+ opts = {
+ "api_key_encoded": "",
+ "days_limit": "365",
+ "variance": "0.01"
+ }
+
+ # Option descriptions
+ optdescs = {
+ "api_key_encoded": "Your Base64-encoded API name/token pair, as provided on your Wigle.net account page.",
+ "days_limit": "Maximum age of data to be considered valid.",
+ "variance": "How tightly to bound queries against the latitude/longitude box extracted from idenified addresses. This value must be between 0.001 and 0.2."
+
+ }
+
+ # Be sure to completely clear any class variables in setup()
+ # or you run the risk of data persisting between scan runs.
+
+ results = dict()
+
+ def setup(self, sfc, userOpts=dict()):
+ self.sf = sfc
+ self.results = dict()
+
+ # Clear / reset any other class member variables here
+ # or you risk them persisting between threads.
+
+ for opt in userOpts.keys():
+ self.opts[opt] = userOpts[opt]
+
+ # What events is this module interested in for input
+ def watchedEvents(self):
+ return ["PHYSICAL_ADDRESS"]
+
+ # What events this module produces
+ def producedEvents(self):
+ return ["WIFI_ACCESS_POINT"]
+
+ def getcoords(self, qry):
+ url = "https://api.wigle.net/api/v2/network/geocode?" + \
+ urllib.urlencode({'addresscode': unicode.encode(qry, 'utf-8', errors='replace')})
+ hdrs = {
+ "Accept": "application/json",
+ "Authorization": "Basic " + self.opts['api_key_encoded']
+ }
+
+ res = self.sf.fetchUrl(url, timeout=30,
+ useragent="SpiderFoot", headers=hdrs)
+ if res['code'] == "404":
+ return None
+ if "too many queries" in res['content']:
+ self.sf.error("Wigle.net query limit reached for the day.", False)
+ return None
+
+ try:
+ info = json.loads(res['content'])
+ if len(info.get('results', [])) == 0:
+ return None
+ return info['results'][0]['boundingbox']
+ except Exception as e:
+ self.sf.error("Error processing JSON response from Wigle.net: " + str(e), False)
+ return None
+
+ def getnetworks(self, coords):
+ url = "https://api.wigle.net/api/v2/network/search?onlymine=false&" + \
+ "latrange1=" + str(coords[0]) + "&latrange2=" + str(coords[1]) + \
+ "&longrange1=" + str(coords[2]) + "&longrange2=" + str(coords[3]) + \
+ "&freenet=false&paynet=false&variance=" + self.opts['variance']
+
+ if self.opts['days_limit'] != "0":
+ dt = datetime.datetime.now() - datetime.timedelta(days=int(self.opts['days_limit']))
+ date_calc = dt.strftime("%Y%m%d")
+ url += "&lastupdt=" + date_calc
+
+ hdrs = {
+ "Accept": "application/json",
+ "Authorization": "Basic " + self.opts['api_key_encoded']
+ }
+
+ res = self.sf.fetchUrl(url, timeout=30,
+ useragent="SpiderFoot", headers=hdrs)
+ if res['code'] == "404":
+ return None
+ if "too many queries" in res['content']:
+ self.sf.error("Wigle.net query limit reached for the day.", False)
+ return None
+
+ ret = list()
+ try:
+ info = json.loads(res['content'])
+ if len(info.get('results', [])) == 0:
+ return None
+ for r in info['results']:
+ if None not in [r['ssid'], r['netid']]:
+ ret.append(r['ssid'] + " (Net ID: " + r['netid'] + ")")
+ return ret
+ except Exception as e:
+ self.sf.error("Error processing JSON response from Wigle.net: " + str(e), False)
+ return None
+
+
+ # Handle events sent to this module
+ def handleEvent(self, event):
+ eventName = event.eventType
+ srcModuleName = event.module
+ eventData = event.data
+
+ self.sf.debug("Received event, " + eventName + ", from " + srcModuleName)
+
+ # Don't look up stuff twice
+ if eventData in self.results:
+ self.sf.debug("Skipping " + eventData + " as already mapped.")
+ return None
+ else:
+ self.results[eventData] = True
+
+ coords = self.getcoords(eventData)
+ if not coords:
+ self.sf.error("Couldn't get coordinates for address from Wigle.net.", False)
+ return None
+
+ nets = self.getnetworks(coords)
+ if not nets:
+ self.sf.error("Couldn't get networks for coordinates from Wigle.net.", False)
+ return None
+
+ for n in nets:
+ e = SpiderFootEvent("WIFI_ACCESS_POINT", n, self.__name__, event)
+ self.notifyListeners(e)
+
+# End of sfp_wigle class
diff --git a/modules/sfp_wikipediaedits.py b/modules/sfp_wikipediaedits.py
new file mode 100644
index 00000000..8ff771ca
--- /dev/null
+++ b/modules/sfp_wikipediaedits.py
@@ -0,0 +1,110 @@
+#-------------------------------------------------------------------------------
+# Name: sfp_wikipediaedits
+# Purpose: Identify edits to Wikipedia articles made from a given IP address
+# or username.
+#
+# Author: Steve Micallef <[email protected]>
+#
+# Created: 10/09/2017
+# Copyright: (c) Steve Micallef
+# Licence: GPL
+#-------------------------------------------------------------------------------
+
+import sys
+import json
+import datetime
+import urllib2
+import re
+from HTMLParser import HTMLParser
+from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+
+class sfp_wikipediaedits(SpiderFootPlugin):
+ """Wikipedia Edits:Footprint,Investigate,Passive:Secondary Networks::Identify edits to Wikipedia articles made from a given IP address or username."""
+
+
+ # Default options
+ opts = {
+ "days_limit": "365"
+ }
+
+ # Option descriptions
+ optdescs = {
+ "days_limit": "Maximum age of data to be considered valid (0 = unlimited)."
+ }
+
+ # Be sure to completely clear any class variables in setup()
+ # or you run the risk of data persisting between scan runs.
+
+ results = dict()
+
+ def setup(self, sfc, userOpts=dict()):
+ self.sf = sfc
+ self.results = dict()
+ self.__dataSource__ = "Wikipedia"
+
+ # Clear / reset any other class member variables here
+ # or you risk them persisting between threads.
+
+ for opt in userOpts.keys():
+ self.opts[opt] = userOpts[opt]
+
+ # What events is this module interested in for input
+ def watchedEvents(self):
+ return ["IP_ADDRESS", "USERNAME"]
+
+ # What events this module produces
+ def producedEvents(self):
+ return ["WIKIPEDIA_PAGE_EDIT"]
+
+ def query(self, qry):
+ url = "https://en.wikipedia.org/w/api.php?action=feedcontributions&user=" + qry
+ if self.opts['days_limit'] != "0":
+ dt = datetime.datetime.now() - datetime.timedelta(days=int(self.opts['days_limit']))
+ y = dt.strftime("%Y")
+ m = dt.strftime("%m")
+ url += "&year=" + y + "&month=" + m
+ res = self.sf.fetchUrl(url, timeout=self.opts['_fetchtimeout'],
+ useragent="SpiderFoot")
+ if res['code'] in [ "404", "403", "500" ]:
+ return None
+
+ links = list()
+ try:
+ parser = HTMLParser()
+ for line in res['content'].split("\n"):
+ matches = re.findall("<link>(.*?)</link>", line, re.IGNORECASE)
+ for m in matches:
+ if "Special:Contributions" in m:
+ continue
+ d = parser.unescape(m)
+ links.append(d)
+ return links
+ except Exception as e:
+ self.sf.error("Error processing response from Wikipedia: " + str(e), False)
+ return None
+
+ # Handle events sent to this module
+ def handleEvent(self, event):
+ eventName = event.eventType
+ srcModuleName = event.module
+ eventData = event.data
+
+ self.sf.debug("Received event, " + eventName + ", from " + srcModuleName)
+
+ # Don't look up stuff twice
+ if eventData in self.results:
+ self.sf.debug("Skipping " + eventData + " as already mapped.")
+ return None
+ else:
+ self.results[eventData] = True
+
+ data = self.query(eventData)
+ if data == None:
+ return None
+
+ for l in data:
+ e = SpiderFootEvent("WIKIPEDIA_PAGE_EDIT", l,
+ self.__name__, event)
+ self.notifyListeners(e)
+
+# End of sfp_wikipediaedits class
diff --git a/sfdb.py b/sfdb.py
index 25b2fd3f..1a32066b 100644
--- a/sfdb.py
+++ b/sfdb.py
@@ -206,7 +206,9 @@ class SpiderFootDb:
"INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('WEBSERVER_BANNER', 'Web Server', 0, 'DATA')",
"INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('WEBSERVER_HTTPHEADERS', 'HTTP Headers', 1, 'DATA')",
"INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('WEBSERVER_STRANGEHEADER', 'Non-Standard HTTP Header', 0, 'DATA')",
- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('WEBSERVER_TECHNOLOGY', 'Web Technology', 0, 'DESCRIPTOR')"
+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('WEBSERVER_TECHNOLOGY', 'Web Technology', 0, 'DESCRIPTOR')",
+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('WIFI_ACCESS_POINT', 'WiFi Access Point Nearby', 0, 'ENTITY')",
+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('WIKIPEDIA_PAGE_EDIT', 'Wikipedia Page Edit', 0, 'DESCRIPTOR')"
]
def __init__(self, opts):