Skip to content

SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.

archive.org module to identify historic pages.

Steve Micallefae3df6ff6859
diff --git a/evtypes.sql b/evtypes.sql
index 0624efa5..82509043 100644
--- a/evtypes.sql
+++ b/evtypes.sql
@@ -94,6 +94,14 @@ INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES (
INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('URL_STATIC', 'URL (Purely Static)', 0, "DESCRIPTOR");
INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('URL_PASSWORD', 'URL (Accepts Passwords)', 0, "DESCRIPTOR");
INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('URL_UPLOAD', 'URL (Accepts Uploads)', 0, "DESCRIPTOR");
+INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('URL_FORM_HISTORIC', 'Historic URL (Form)', 0, "DESCRIPTOR");
+INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('URL_FLASH_HISTORIC', 'Historic URL (Uses Flash)', 0, "DESCRIPTOR");
+INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('URL_JAVASCRIPT_HISTORIC', 'Historic URL (Uses Javascript)', 0, "DESCRIPTOR");
+INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('URL_WEB_FRAMEWORK_HISTORIC', 'Historic URL (Uses a Web Framework)', 0, "DESCRIPTOR");
+INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('URL_JAVA_APPLET_HISTORIC', 'Historic URL (Uses Java Applet)', 0, "DESCRIPTOR");
+INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('URL_STATIC_HISTORIC', 'Historic URL (Purely Static)', 0, "DESCRIPTOR");
+INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('URL_PASSWORD_HISTORIC', 'Historic URL (Accepts Passwords)', 0, "DESCRIPTOR");
+INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('URL_UPLOAD_HISTORIC', 'Historic URL (Accepts Uploads)', 0, "DESCRIPTOR");
INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('WEBSERVER_BANNER', 'Web Server', 0, "DATA");
INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('WEBSERVER_HTTPHEADERS', 'HTTP Headers', 1, "DATA");
INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('WEBSERVER_STRANGEHEADER', 'Non-Standard HTTP Header', 0, "DATA");
diff --git a/modules/sfp_historic.py b/modules/sfp_historic.py
new file mode 100644
index 00000000..5258cadf
--- /dev/null
+++ b/modules/sfp_historic.py
@@ -0,0 +1,147 @@
+# -*- coding: utf-8 -*-
+# -------------------------------------------------------------------------------
+# Name: sfp_historic
+# Purpose: Queries archive.org (Wayback machine) for historic versions of
+# certain pages.
+#
+# Author: Steve Micallef <[email protected]>
+#
+# Created: 20/07/2015
+# Copyright: (c) Steve Micallef 2015
+# Licence: GPL
+# -------------------------------------------------------------------------------
+
+import re
+import urllib
+import datetime
+import time
+import json
+from sflib import SpiderFoot, SpiderFootPlugin, SpiderFootEvent
+
+
+class sfp_historic(SpiderFootPlugin):
+ """Historic Files:Identifies historic versions of interesting files/pages from the Wayback Machine."""
+
+ # Default options
+ opts = {
+ 'farback': "30,60,90",
+ 'intfiles': True,
+ 'passwordpages': True,
+ 'formpages': False,
+ 'flashpages': False,
+ 'javapages': False,
+ 'staticpages': False,
+ 'uploadpages': False,
+ 'webframeworkpages': False,
+ 'javascriptpages': False
+ }
+
+ # Option descriptions
+ optdescs = {
+ 'farback': "Number of days back to look for older versions of files/pages in the Wayback Machine snapshots. Comma-separate the values, so for example 30,60,90 means to look for snapshots 30 days, 60 days and 90 days back.",
+ 'intfiles': "Query the Wayback Machine for historic versions of Interesting Files.",
+ 'passwordpages': "Query the Wayback Machine for historic versions of URLs with passwords.",
+ 'formpages': "Query the Wayback Machine for historic versions of URLs with forms.",
+ 'uploadpages': "Query the Wayback Machine for historic versions of URLs accepting uploads.",
+ 'flashpages': "Query the Wayback Machine for historic versions of URLs containing Flash.",
+ 'javapages': "Query the Wayback Machine for historic versions of URLs using Java Applets.",
+ 'staticpages': "Query the Wayback Machine for historic versions of purely static URLs.",
+ "webframeworkpages": "Query the Wayback Machine for historic versions of URLs using Javascript frameworks.",
+ "javascriptpages": "Query the Wayback Machine for historic versions of URLs using Javascript."
+ }
+
+ results = list()
+ foundDates = list()
+
+ def setup(self, sfc, userOpts=dict()):
+ self.sf = sfc
+ self.results = list()
+ self.foundDates = list()
+
+ for opt in userOpts.keys():
+ self.opts[opt] = userOpts[opt]
+
+ # What events is this module interested in for input
+ def watchedEvents(self):
+ return ["INTERESTING_FILE", "URL_PASSWORD", "URL_FORM", "URL_FLASH",
+ "URL_STATIC", "URL_JAVA_APPLET", "URL_UPLOAD", "URL_JAVASCRIPT",
+ "URL_WEB_FRAMEWORK"]
+
+ # What events this module produces
+ # This is to support the end user in selecting modules based on events
+ # produced.
+ def producedEvents(self):
+ return ["INTERESTING_FILE_HISTORIC", "URL_PASSWORD_HISTORIC",
+ "URL_FORM_HISTORIC", "URL_FLASH_HISTORIC",
+ "URL_STATIC_HISTORIC", "URL_JAVA_APPLET_HISTORIC",
+ "URL_UPLOAD_HISTORIC", "URL_WEB_FRAMEWORK_HISTORIC",
+ "URL_JAVASCRIPT_HISTORIC"]
+
+ # Handle events sent to this module
+ def handleEvent(self, event):
+ eventName = event.eventType
+ srcModuleName = event.module
+ eventData = event.data
+
+ self.sf.debug("Received event, " + eventName + ", from " + srcModuleName)
+
+ if eventName == "INTERESTING_FILE" and not self.opts['intfiles']:
+ return None
+ if eventName == "URL_PASSWORD" and not self.opts['passwordpages']:
+ return None
+ if eventName == "URL_STATIC" and not self.opts['staticpages']:
+ return None
+ if eventName == "URL_FORM" and not self.opts['formpages']:
+ return None
+ if eventName == "URL_UPLOAD" and not self.opts['uploadpages']:
+ return None
+ if eventName == "URL_JAVA_APPLET" and not self.opts['javapages']:
+ return None
+ if eventName == "URL_FLASH" and not self.opts['flashpages']:
+ return None
+ if eventName == "URL_JAVASCRIPT" and not self.opts['javascriptpages']:
+ return None
+ if eventName == "URL_WEB_FRAMEWORK" and not self.opts['webframeworkpages']:
+ return None
+
+ if eventData in self.results:
+ return None
+ else:
+ self.results.append(eventData)
+
+ for daysback in self.opts['farback'].split(","):
+ newDate = datetime.datetime.now() - datetime.timedelta(days=int(daysback))
+ maxDate = newDate.strftime("%Y%m%d")
+
+ url = "http://archive.org/wayback/available?url=" + eventData + \
+ "&timestamp=" + maxDate
+ res = self.sf.fetchUrl(url, timeout=self.opts['_fetchtimeout'],
+ useragent=self.opts['_useragent'])
+
+ if res == None:
+ self.sf.error("Unable to fetch " + url, False)
+ continue
+
+ ret = json.loads(res['content'])
+ if ret == None:
+ self.sf.error("Unable to process empty response from archive.org: " + \
+ eventData, False)
+ continue
+
+ if len(ret['archived_snapshots']) < 1:
+ self.sf.debug("No archived snapshots for " + eventData)
+ continue
+
+ wbmlink = ret['archived_snapshots']['closest']['url']
+ if wbmlink in self.foundDates:
+ self.sf.debug("Snapshot already fetched.")
+ continue
+
+ self.foundDates.append(wbmlink)
+ name = eventName + "_HISTORIC"
+
+ self.sf.info("Found a historic file: " + wbmlink)
+ evt = SpiderFootEvent(name, wbmlink, self.__name__, event)
+ self.notifyListeners(evt)
+
+# End of sfp_historic class
diff --git a/sfdb.py b/sfdb.py
index 2d49afb6..e58b2038 100644
--- a/sfdb.py
+++ b/sfdb.py
@@ -168,6 +168,14 @@ class SpiderFootDb:
"INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('URL_STATIC', 'URL (Purely Static)', 0, 'DESCRIPTOR')",
"INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('URL_PASSWORD', 'URL (Accepts Passwords)', 0, 'DESCRIPTOR')",
"INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('URL_UPLOAD', 'URL (Accepts Uploads)', 0, 'DESCRIPTOR')",
+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('URL_FORM_HISTORIC', 'Historic URL (Form)', 0, 'DESCRIPTOR')",
+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('URL_FLASH_HISTORIC', 'Historic URL (Uses Flash)', 0, 'DESCRIPTOR')",
+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('URL_JAVASCRIPT_HISTORIC', 'Historic URL (Uses Javascript)', 0, 'DESCRIPTOR')",
+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('URL_WEB_FRAMEWORK_HISTORIC', 'Historic URL (Uses a Web Framework)', 0, 'DESCRIPTOR')",
+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('URL_JAVA_APPLET_HISTORIC', 'Historic URL (Uses Java Applet)', 0, 'DESCRIPTOR')",
+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('URL_STATIC_HISTORIC', 'Historic URL (Purely Static)', 0, 'DESCRIPTOR')",
+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('URL_PASSWORD_HISTORIC', 'Historic URL (Accepts Passwords)', 0, 'DESCRIPTOR')",
+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('URL_UPLOAD_HISTORIC', 'Historic URL (Accepts Uploads)', 0, 'DESCRIPTOR')",
"INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('WEBSERVER_BANNER', 'Web Server', 0, 'DATA')",
"INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('WEBSERVER_HTTPHEADERS', 'HTTP Headers', 1, 'DATA')",
"INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('WEBSERVER_STRANGEHEADER', 'Non-Standard HTTP Header', 0, 'DATA')",