SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.
Python attacksurfacecticybersecurityfootprintinghackinginformation-gatheringinformation-securityinfosecintelligence-gatheringosintosint-frameworkosint-reconnaissanceosint-toolpentestingpythonreconsecurity-toolsthreat-intelligencethreatintel
dos2unix sfdb.py
Brendan Colesdcb65d44af1a
diff --git a/sfdb.py b/sfdb.pyindex eca1e5eb..c88ad331 100644--- a/sfdb.py+++ b/sfdb.py@@ -1,882 +1,882 @@-# -*- coding: utf-8 -*--# --------------------------------------------------------------------------------# Name: sfdb-# Purpose: Common functions for working with the database back-end.-#-# Author: Steve Micallef <[email protected]>-#-# Created: 15/05/2012-# Copyright: (c) Steve Micallef 2012-# Licence: GPL-# ---------------------------------------------------------------------------------import sqlite3-import re-import time-from sflib import SpiderFoot--# SQLite doesn't support regex queries, so we create-# a custom function to do so..-def __dbregex__(qry, data):- try:- rx = re.compile(qry, re.IGNORECASE|re.DOTALL)- ret = rx.match(data)- except BaseException as e:- return False- return ret is not None---class SpiderFootDb:- sf = None- dbh = None- conn = None-- # Queries for creating the SpiderFoot database- createSchemaQueries = [- "PRAGMA journal_mode=WAL",- "CREATE TABLE tbl_event_types ( \- event VARCHAR NOT NULL PRIMARY KEY, \- event_descr VARCHAR NOT NULL, \- event_raw INT NOT NULL DEFAULT 0, \- event_type VARCHAR NOT NULL \- )",- "CREATE TABLE tbl_config ( \- scope VARCHAR NOT NULL, \- opt VARCHAR NOT NULL, \- val VARCHAR NOT NULL, \- PRIMARY KEY (scope, opt) \- )",- "CREATE TABLE tbl_scan_instance ( \- guid VARCHAR NOT NULL PRIMARY KEY, \- name VARCHAR NOT NULL, \- seed_target VARCHAR NOT NULL, \- created INT DEFAULT 0, \- started INT DEFAULT 0, \- ended INT DEFAULT 0, \- status VARCHAR NOT NULL \- )",- "CREATE TABLE tbl_scan_log ( \- scan_instance_id VARCHAR NOT NULL REFERENCES tbl_scan_instance(guid), \- generated INT NOT NULL, \- component VARCHAR, \- type VARCHAR NOT NULL, \- message VARCHAR \- )",- "CREATE TABLE tbl_scan_config ( \- scan_instance_id VARCHAR NOT NULL REFERENCES tbl_scan_instance(guid), \- component VARCHAR NOT NULL, \- opt VARCHAR NOT NULL, \- val VARCHAR NOT NULL \- )",- "CREATE TABLE tbl_scan_results ( \- scan_instance_id VARCHAR NOT NULL REFERENCES tbl_scan_instance(guid), \- hash VARCHAR NOT NULL, \- type VARCHAR NOT NULL REFERENCES tbl_event_types(event), \- generated INT NOT NULL, \- confidence INT NOT NULL DEFAULT 100, \- visibility INT NOT NULL DEFAULT 100, \- risk INT NOT NULL DEFAULT 0, \- module VARCHAR NOT NULL, \- data VARCHAR, \- false_positive INT NOT NULL DEFAULT 0, \- source_event_hash VARCHAR DEFAULT 'ROOT' \- )",- "CREATE INDEX idx_scan_results_id ON tbl_scan_results (scan_instance_id)",- "CREATE INDEX idx_scan_results_type ON tbl_scan_results (scan_instance_id, type)",- "CREATE INDEX idx_scan_results_hash ON tbl_scan_results (scan_instance_id, hash)",- "CREATE INDEX idx_scan_results_srchash ON tbl_scan_results (scan_instance_id, source_event_hash)",- "CREATE INDEX idx_scan_logs ON tbl_scan_log (scan_instance_id)"-- ]-- createTypeQueries = [- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('ROOT', 'Internal SpiderFoot Root event', 1, 'INTERNAL')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('ACCOUNT_EXTERNAL_OWNED', 'Account on External Site', 0, 'ENTITY')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('ACCOUNT_EXTERNAL_OWNED_COMPROMISED', 'Hacked Account on External Site', 0, 'DESCRIPTOR')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('ACCOUNT_EXTERNAL_USER_SHARED_COMPROMISED', 'Hacked User Account on External Site', 0, 'DESCRIPTOR')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('AFFILIATE_EMAILADDR', 'Affiliate - Email Address', 0, 'ENTITY')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('AFFILIATE_INTERNET_NAME', 'Affiliate - Internet Name', 0, 'ENTITY')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('AFFILIATE_IPADDR', 'Affiliate - IP Address', 0, 'ENTITY')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('AFFILIATE_WEB_CONTENT', 'Affiliate - Web Content', 1, 'DATA')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('AFFILIATE_DOMAIN', 'Affiliate - Domain Name', 0, 'ENTITY')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('AFFILIATE_COMPANY_NAME', 'Affiliate - Company Name', 0, 'ENTITY')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('AFFILIATE_DOMAIN_WHOIS', 'Affiliate - Domain Whois', 1, 'DATA')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('AFFILIATE_DESCRIPTION_CATEGORY', 'Affiliate Description - Category', 0, 'DESCRIPTOR')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('AFFILIATE_DESCRIPTION_ABSTRACT', 'Affiliate Description - Abstract', 0, 'DESCRIPTOR')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('APPSTORE_ENTRY', 'App Store Entry', 0, 'ENTITY')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('CLOUD_STORAGE_BUCKET', 'Cloud Storage Bucket', 0, 'ENTITY')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('CLOUD_STORAGE_BUCKET_OPEN', 'Cloud Storage Bucket Open', 0, 'DESCRIPTOR')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('COMPANY_NAME', 'Company Name', 0, 'ENTITY')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('BASE64_DATA', 'Base64-encoded Data', 1, 'DATA')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('BITCOIN_ADDRESS', 'Bitcoin Address', 0, 'ENTITY')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('BITCOIN_BALANCE', 'Bitcoin Balance', 0, 'DESCRIPTOR')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('BGP_AS_OWNER', 'BGP AS Ownership', 0, 'ENTITY')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('BGP_AS_MEMBER', 'BGP AS Membership', 0, 'ENTITY')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('BGP_AS_PEER', 'BGP AS Peer', 0, 'ENTITY')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('BLACKLISTED_IPADDR', 'Blacklisted IP Address', 0, 'DESCRIPTOR')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('BLACKLISTED_AFFILIATE_IPADDR', 'Blacklisted Affiliate IP Address', 0, 'DESCRIPTOR')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('BLACKLISTED_SUBNET', 'Blacklisted IP on Same Subnet', 0, 'DESCRIPTOR')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('BLACKLISTED_NETBLOCK', 'Blacklisted IP on Owned Netblock', 0, 'DESCRIPTOR')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('CO_HOSTED_SITE', 'Co-Hosted Site', 0, 'ENTITY')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('CO_HOSTED_SITE_DOMAIN', 'Co-Hosted Site - Domain Name', 0, 'ENTITY')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('CO_HOSTED_SITE_DOMAIN_WHOIS', 'Co-Hosted Site - Domain Whois', 1, 'DATA')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('DARKNET_MENTION_URL', 'Darknet Mention URL', 0, 'DESCRIPTOR')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('DARKNET_MENTION_CONTENT', 'Darknet Mention Web Content', 1, 'DATA')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('DATE_HUMAN_DOB', 'Date of Birth', 0, 'ENTITY')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('DEFACED_INTERNET_NAME', 'Defaced', 0, 'DESCRIPTOR')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('DEFACED_IPADDR', 'Defaced IP Address', 0, 'DESCRIPTOR')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('DEFACED_AFFILIATE_INTERNET_NAME', 'Defaced Affiliate', 0, 'DESCRIPTOR')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('DEFACED_COHOST', 'Defaced Co-Hosted Site', 0, 'DESCRIPTOR')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('DEFACED_AFFILIATE_IPADDR', 'Defaced Affiliate IP Address', 0, 'DESCRIPTOR')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('DESCRIPTION_CATEGORY', 'Description - Category', 0, 'DESCRIPTOR')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('DESCRIPTION_ABSTRACT', 'Description - Abstract', 0, 'DESCRIPTOR')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('DEVICE_TYPE', 'Device Type', 0, 'DESCRIPTOR')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('DNS_TEXT', 'DNS TXT Record', 0, 'DATA')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('DNS_SRV', 'DNS SRV Record', 0, 'DATA')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('DNS_SPF', 'DNS SPF Record', 0, 'DATA')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('DOMAIN_NAME', 'Domain Name', 0, 'ENTITY')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('DOMAIN_NAME_PARENT', 'Domain Name (Parent)', 0, 'ENTITY')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('DOMAIN_REGISTRAR', 'Domain Registrar', 0, 'ENTITY')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('DOMAIN_WHOIS', 'Domain Whois', 1, 'DATA')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('EMAILADDR', 'Email Address', 0, 'ENTITY')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('EMAILADDR_COMPROMISED', 'Hacked Email Address', 0, 'DESCRIPTOR')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('ERROR_MESSAGE', 'Error Message', 0, 'DATA')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('ETHEREUM_ADDRESS', 'Ethereum Address', 0, 'ENTITY')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('GEOINFO', 'Physical Location', 0, 'DESCRIPTOR')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('HTTP_CODE', 'HTTP Status Code', 0, 'DATA')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('HUMAN_NAME', 'Human Name', 0, 'ENTITY')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('INTERESTING_FILE', 'Interesting File', 0, 'DESCRIPTOR')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('INTERESTING_FILE_HISTORIC', 'Historic Interesting File', 0, 'DESCRIPTOR')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('JUNK_FILE', 'Junk File', 0, 'DESCRIPTOR')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('INTERNET_NAME', 'Internet Name', 0, 'ENTITY')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('INTERNET_NAME_UNRESOLVED', 'Internet Name - Unresolved', 0, 'ENTITY')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('IP_ADDRESS', 'IP Address', 0, 'ENTITY')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('IPV6_ADDRESS', 'IPv6 Address', 0, 'ENTITY')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('LINKED_URL_INTERNAL', 'Linked URL - Internal', 0, 'SUBENTITY')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('LINKED_URL_EXTERNAL', 'Linked URL - External', 0, 'SUBENTITY')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('MALICIOUS_ASN', 'Malicious AS', 0, 'DESCRIPTOR')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('MALICIOUS_IPADDR', 'Malicious IP Address', 0, 'DESCRIPTOR')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('MALICIOUS_COHOST', 'Malicious Co-Hosted Site', 0, 'DESCRIPTOR')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('MALICIOUS_EMAILADDR', 'Malicious E-mail Address', 0, 'DESCRIPTOR')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('MALICIOUS_INTERNET_NAME', 'Malicious Internet Name', 0, 'DESCRIPTOR')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('MALICIOUS_AFFILIATE_INTERNET_NAME', 'Malicious Affiliate', 0, 'DESCRIPTOR')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('MALICIOUS_AFFILIATE_IPADDR', 'Malicious Affiliate IP Address', 0, 'DESCRIPTOR')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('MALICIOUS_NETBLOCK', 'Malicious IP on Owned Netblock', 0, 'DESCRIPTOR')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('MALICIOUS_PHONE_NUMBER', 'Malicious Phone Number', 0, 'DESCRIPTOR')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('MALICIOUS_SUBNET', 'Malicious IP on Same Subnet', 0, 'DESCRIPTOR')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('NETBLOCK_OWNER', 'Netblock Ownership', 0, 'ENTITY')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('NETBLOCK_MEMBER', 'Netblock Membership', 0, 'ENTITY')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('NETBLOCK_WHOIS', 'Netblock Whois', 1, 'DATA')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('OPERATING_SYSTEM', 'Operating System', 0, 'DESCRIPTOR')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('LEAKSITE_URL', 'Leak Site URL', 0, 'ENTITY')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('LEAKSITE_CONTENT', 'Leak Site Content', 1, 'DATA')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('PHONE_NUMBER', 'Phone Number', 0, 'ENTITY')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('PHYSICAL_ADDRESS', 'Physical Address', 0, 'ENTITY')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('PHYSICAL_COORDINATES', 'Physical Coordinates', 0, 'ENTITY')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('PGP_KEY', 'PGP Public Key', 0, 'DATA')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('PROVIDER_DNS', 'Name Server (DNS ''NS'' Records)', 0, 'ENTITY')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('PROVIDER_JAVASCRIPT', 'Externally Hosted Javascript', 0, 'ENTITY')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('PROVIDER_MAIL', 'Email Gateway (DNS ''MX'' Records)', 0, 'ENTITY')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('PROVIDER_HOSTING', 'Hosting Provider', 0, 'ENTITY')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('PROVIDER_TELCO', 'Telecommunications Provider', 0, 'ENTITY')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('PUBLIC_CODE_REPO', 'Public Code Repository', 0, 'ENTITY')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('RAW_RIR_DATA', 'Raw Data from RIRs/APIs', 1, 'DATA')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('RAW_DNS_RECORDS', 'Raw DNS Records', 1, 'DATA')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('RAW_FILE_META_DATA', 'Raw File Meta Data', 1, 'DATA')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('SEARCH_ENGINE_WEB_CONTENT', 'Search Engine''s Web Content', 1, 'DATA')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('SOCIAL_MEDIA', 'Social Media Presence', 0, 'ENTITY')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('SIMILARDOMAIN', 'Similar Domain', 0, 'ENTITY')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('SIMILARDOMAIN_WHOIS', 'Similar Domain - Whois', 1, 'DATA')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('SOFTWARE_USED', 'Software Used', 0, 'SUBENTITY')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('SSL_CERTIFICATE_RAW', 'SSL Certificate - Raw Data', 1, 'DATA')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('SSL_CERTIFICATE_ISSUED', 'SSL Certificate - Issued to', 0, 'ENTITY')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('SSL_CERTIFICATE_ISSUER', 'SSL Certificate - Issued by', 0, 'ENTITY')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('SSL_CERTIFICATE_MISMATCH', 'SSL Certificate Host Mismatch', 0, 'DESCRIPTOR')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('SSL_CERTIFICATE_EXPIRED', 'SSL Certificate Expired', 0, 'DESCRIPTOR')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('SSL_CERTIFICATE_EXPIRING', 'SSL Certificate Expiring', 0, 'DESCRIPTOR')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('TARGET_WEB_CONTENT', 'Web Content', 1, 'DATA')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('TARGET_WEB_CONTENT_TYPE', 'Web Content Type', 0, 'DESCRIPTOR')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('TARGET_WEB_COOKIE', 'Cookies', 0, 'DATA')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('TCP_PORT_OPEN', 'Open TCP Port', 0, 'SUBENTITY')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('TCP_PORT_OPEN_BANNER', 'Open TCP Port Banner', 0, 'DATA')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('UDP_PORT_OPEN', 'Open UDP Port', 0, 'SUBENTITY')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('UDP_PORT_OPEN_INFO', 'Open UDP Port Information', 0, 'DATA')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('URL_ADBLOCKED_EXTERNAL', 'URL (AdBlocked External)', 0, 'DESCRIPTOR')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('URL_ADBLOCKED_INTERNAL', 'URL (AdBlocked Internal)', 0, 'DESCRIPTOR')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('URL_FORM', 'URL (Form)', 0, 'DESCRIPTOR')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('URL_FLASH', 'URL (Uses Flash)', 0, 'DESCRIPTOR')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('URL_JAVASCRIPT', 'URL (Uses Javascript)', 0, 'DESCRIPTOR')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('URL_WEB_FRAMEWORK', 'URL (Uses a Web Framework)', 0, 'DESCRIPTOR')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('URL_JAVA_APPLET', 'URL (Uses Java Applet)', 0, 'DESCRIPTOR')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('URL_STATIC', 'URL (Purely Static)', 0, 'DESCRIPTOR')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('URL_PASSWORD', 'URL (Accepts Passwords)', 0, 'DESCRIPTOR')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('URL_UPLOAD', 'URL (Accepts Uploads)', 0, 'DESCRIPTOR')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('URL_FORM_HISTORIC', 'Historic URL (Form)', 0, 'DESCRIPTOR')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('URL_FLASH_HISTORIC', 'Historic URL (Uses Flash)', 0, 'DESCRIPTOR')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('URL_JAVASCRIPT_HISTORIC', 'Historic URL (Uses Javascript)', 0, 'DESCRIPTOR')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('URL_WEB_FRAMEWORK_HISTORIC', 'Historic URL (Uses a Web Framework)', 0, 'DESCRIPTOR')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('URL_JAVA_APPLET_HISTORIC', 'Historic URL (Uses Java Applet)', 0, 'DESCRIPTOR')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('URL_STATIC_HISTORIC', 'Historic URL (Purely Static)', 0, 'DESCRIPTOR')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('URL_PASSWORD_HISTORIC', 'Historic URL (Accepts Passwords)', 0, 'DESCRIPTOR')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('URL_UPLOAD_HISTORIC', 'Historic URL (Accepts Uploads)', 0, 'DESCRIPTOR')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('USERNAME', 'Username', 0, 'ENTITY')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('VULNERABILITY', 'Vulnerability in Public Domain', 0, 'DESCRIPTOR')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('WEB_ANALYTICS_ID', 'Web Analytics', 0, 'ENTITY')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('WEBSERVER_BANNER', 'Web Server', 0, 'DATA')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('WEBSERVER_HTTPHEADERS', 'HTTP Headers', 1, 'DATA')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('WEBSERVER_STRANGEHEADER', 'Non-Standard HTTP Header', 0, 'DATA')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('WEBSERVER_TECHNOLOGY', 'Web Technology', 0, 'DESCRIPTOR')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('WIFI_ACCESS_POINT', 'WiFi Access Point Nearby', 0, 'ENTITY')",- "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('WIKIPEDIA_PAGE_EDIT', 'Wikipedia Page Edit', 0, 'DESCRIPTOR')"- ]-- def __init__(self, opts, init=False):- self.sf = SpiderFoot(opts)-- # connect() will create the database file if it doesn't exist, but- # at least we can use this opportunity to ensure we have permissions to- # read and write to such a file.- dbh = sqlite3.connect(self.sf.myPath() + "/" + opts['__database'], timeout=10)- if dbh is None:- self.sf.fatal("Could not connect to internal database, and couldn't create " + opts['__database'])- dbh.text_factory = str-- self.conn = dbh- self.dbh = dbh.cursor()-- # Now we actually check to ensure the database file has the schema set- # up correctly.- try:- self.dbh.execute('SELECT COUNT(*) FROM tbl_scan_config')- self.conn.create_function("REGEXP", 2, __dbregex__)- except sqlite3.Error:- # .. If not set up, we set it up.- try:- self.create()- init = True- except BaseException as e:- self.sf.error("Tried to set up the SpiderFoot database schema, but failed: " + e.args[0])- return-- if init:- print("Attempting to verify database and update if necessary...")- for qry in self.createTypeQueries:- try:- self.dbh.execute(qry)- self.conn.commit()- except BaseException as e:- continue- self.conn.commit()- #self.conn.close()-- #- # Back-end database operations- #-- # Create the back-end schema- def create(self):- try:- for qry in self.createSchemaQueries:- self.dbh.execute(qry)- self.conn.commit()- for qry in self.createTypeQueries:- self.dbh.execute(qry)- self.conn.commit()- except sqlite3.Error as e:- raise BaseException("SQL error encountered when setting up database: " + e.args[0])-- # Close the database handle- def close(self):- self.dbh.close()-- # Search results- # criteria is search criteria such as:- # - scan_id (search within a scan, if omitted search all)- # - type (search a specific type, if omitted search all)- # - value (search values for a specific string, if omitted search all)- # - regex (search values for a regular expression)- # ** at least two criteria must be set **- def search(self, criteria, filterFp=False):- if criteria.values().count(None) == 3:- return False-- qvars = list()- qry = "SELECT ROUND(c.generated) AS generated, c.data, \- s.data as 'source_data', \- c.module, c.type, c.confidence, c.visibility, c.risk, c.hash, \- c.source_event_hash, t.event_descr, t.event_type, c.scan_instance_id, \- c.false_positive as 'fp', s.false_positive as 'parent_fp' \- FROM tbl_scan_results c, tbl_scan_results s, tbl_event_types t \- WHERE s.scan_instance_id = c.scan_instance_id AND \- t.event = c.type AND c.source_event_hash = s.hash "-- if filterFp:- qry += " AND c.false_positive <> 1 "-- if criteria.get('scan_id') is not None:- qry += "AND c.scan_instance_id = ? "- qvars.append(criteria['scan_id'])-- if criteria.get('type') is not None:- qry += " AND c.type = ? "- qvars.append(criteria['type'])-- if criteria.get('value') is not None:- qry += " AND (c.data LIKE ? OR s.data LIKE ?) "- qvars.append(criteria['value'])- qvars.append(criteria['value'])-- if criteria.get('regex') is not None:- qry += " AND (c.data REGEXP ? OR s.data REGEXP ?) "- qvars.append(criteria['regex'])- qvars.append(criteria['regex'])-- qry += " ORDER BY c.data"-- try:- #print(qry)- #print(str(qvars))- self.dbh.execute(qry, qvars)- return self.dbh.fetchall()- except sqlite3.Error as e:- self.sf.error("SQL error encountered when fetching search results: " + e.args[0])-- # Get event types- def eventTypes(self):- qry = "SELECT event_descr, event, event_raw, event_type FROM tbl_event_types"- try:- self.dbh.execute(qry)- return self.dbh.fetchall()- except sqlite3.Error as e:- self.sf.error("SQL error encountered when retreiving event types:" + e.args[0])-- # Log an event to the database- def scanLogEvent(self, instanceId, classification, message, component=None):- if component is None:- component = "SpiderFoot"-- qry = "INSERT INTO tbl_scan_log \- (scan_instance_id, generated, component, type, message) \- VALUES (?, ?, ?, ?, ?)"- try:- self.dbh.execute(qry, (- instanceId, time.time() * 1000, component, classification, message- ))- self.conn.commit()- except sqlite3.Error as e:- if "locked" in e.args[0]:- # TODO: Do something smarter here to handle locked databases- self.sf.fatal("Unable to log event in DB due to lock: " + e.args[0])- else:- self.sf.fatal("Unable to log event in DB: " + e.args[0])-- return True-- # Store a scan instance- def scanInstanceCreate(self, instanceId, scanName, scanTarget):- qry = "INSERT INTO tbl_scan_instance \- (guid, name, seed_target, created, status) \- VALUES (?, ?, ?, ?, ?)"- try:- self.dbh.execute(qry, (- instanceId, scanName, scanTarget, time.time() * 1000, 'CREATED'- ))- self.conn.commit()- except sqlite3.Error as e:- self.sf.fatal("Unable to create instance in DB: " + e.args[0])-- return True-- # Update the start time, end time or status (or all 3) of a scan instance- def scanInstanceSet(self, instanceId, started=None, ended=None, status=None):- qvars = list()- qry = "UPDATE tbl_scan_instance SET "-- if started is not None:- qry += " started = ?,"- qvars.append(started)-- if ended is not None:- qry += " ended = ?,"- qvars.append(ended)-- if status is not None:- qry += " status = ?,"- qvars.append(status)-- # guid = guid is a little hack to avoid messing with , placement above- qry += " guid = guid WHERE guid = ?"- qvars.append(instanceId)-- try:- self.dbh.execute(qry, qvars)- self.conn.commit()- except sqlite3.Error:- self.sf.fatal("Unable to set information for the scan instance.")-- # Return info about a scan instance (name, target, created, started,- # ended, status) - don't need this yet - untested- def scanInstanceGet(self, instanceId):- qry = "SELECT name, seed_target, ROUND(created/1000) AS created, \- ROUND(started/1000) AS started, ROUND(ended/1000) AS ended, status \- FROM tbl_scan_instance WHERE guid = ?"- qvars = [instanceId]- try:- self.dbh.execute(qry, qvars)- return self.dbh.fetchone()- except sqlite3.Error as e:- self.sf.error("SQL error encountered when retreiving scan instance:" + e.args[0])-- # Obtain a summary of the results per event type- def scanResultSummary(self, instanceId, by="type"):- if by == "type":- qry = "SELECT r.type, e.event_descr, MAX(ROUND(generated)) AS last_in, \- count(*) AS total, count(DISTINCT r.data) as utotal FROM \- tbl_scan_results r, tbl_event_types e WHERE e.event = r.type \- AND r.scan_instance_id = ? GROUP BY r.type ORDER BY e.event_descr"-- if by == "module":- qry = "SELECT r.module, '', MAX(ROUND(generated)) AS last_in, \- count(*) AS total, count(DISTINCT r.data) as utotal FROM \- tbl_scan_results r, tbl_event_types e WHERE e.event = r.type \- AND r.scan_instance_id = ? GROUP BY r.module ORDER BY r.module DESC"-- if by == "entity":- qry = "SELECT r.data, e.event_descr, MAX(ROUND(generated)) AS last_in, \- count(*) AS total, count(DISTINCT r.data) as utotal FROM \- tbl_scan_results r, tbl_event_types e WHERE e.event = r.type \- AND r.scan_instance_id = ? \- AND e.event_type in ('ENTITY') \- GROUP BY r.data, e.event_descr ORDER BY total DESC limit 50"-- qvars = [instanceId]- try:- self.dbh.execute(qry, qvars)- return self.dbh.fetchall()- except sqlite3.Error as e:- self.sf.error("SQL error encountered when fetching result summary: " + e.args[0])-- # Obtain the data for a scan and event type- def scanResultEvent(self, instanceId, eventType='ALL', filterFp=False):- qry = "SELECT ROUND(c.generated) AS generated, c.data, \- s.data as 'source_data', \- c.module, c.type, c.confidence, c.visibility, c.risk, c.hash, \- c.source_event_hash, t.event_descr, t.event_type, s.scan_instance_id, \- c.false_positive as 'fp', s.false_positive as 'parent_fp' \- FROM tbl_scan_results c, tbl_scan_results s, tbl_event_types t \- WHERE c.scan_instance_id = ? AND c.source_event_hash = s.hash AND \- s.scan_instance_id = c.scan_instance_id AND \- t.event = c.type"-- qvars = [instanceId]-- if eventType != "ALL":- qry += " AND c.type = ?"- qvars.append(eventType)-- if filterFp:- qry += " AND c.false_positive <> 1"-- qry += " ORDER BY c.data"-- try:- self.dbh.execute(qry, qvars)- return self.dbh.fetchall()- except sqlite3.Error as e:- self.sf.error("SQL error encountered when fetching result events: " + e.args[0])-- # Obtain a unique list of elements- def scanResultEventUnique(self, instanceId, eventType='ALL', filterFp=False):- qry = "SELECT DISTINCT data, type, COUNT(*) FROM tbl_scan_results \- WHERE scan_instance_id = ?"- qvars = [instanceId]-- if eventType != "ALL":- qry += " AND type = ?"- qvars.append(eventType)-- if filterFp:- qry += " AND false_positive <> 1"-- qry += " GROUP BY type, data ORDER BY COUNT(*)"-- try:- self.dbh.execute(qry, qvars)- return self.dbh.fetchall()- except sqlite3.Error as e:- self.sf.error("SQL error encountered when fetching unique result events: " + e.args[0])-- # Get scan logs- def scanLogs(self, instanceId, limit=None, fromRowId=None, reverse=False):- qry = "SELECT generated AS generated, component, \- type, message, rowid FROM tbl_scan_log WHERE scan_instance_id = ?"- if fromRowId:- qry += " and rowid > ?"-- qry += " ORDER BY generated "- if reverse:- qry += "ASC"- else:- qry += "DESC"- qvars = [instanceId]-- if fromRowId:- qvars.append(fromRowId)-- if limit is not None:- qry += " LIMIT ?"- qvars.append(limit)-- try:- self.dbh.execute(qry, qvars)- return self.dbh.fetchall()- except sqlite3.Error as e:- self.sf.error("SQL error encountered when fetching scan logs: " + e.args[0])-- # Get scan errors- def scanErrors(self, instanceId, limit=None):- qry = "SELECT generated AS generated, component, \- message FROM tbl_scan_log WHERE scan_instance_id = ? \- AND type = 'ERROR' ORDER BY generated DESC"- qvars = [instanceId]-- if limit is not None:- qry += " LIMIT ?"- qvars.append(limit)-- try:- self.dbh.execute(qry, qvars)- return self.dbh.fetchall()- except sqlite3.Error as e:- self.sf.error("SQL error encountered when fetching scan errors: " + e.args[0])-- # Delete a scan instance- def scanInstanceDelete(self, instanceId):- qry1 = "DELETE FROM tbl_scan_instance WHERE guid = ?"- qry2 = "DELETE FROM tbl_scan_config WHERE scan_instance_id = ?"- qry3 = "DELETE FROM tbl_scan_results WHERE scan_instance_id = ?"- qry4 = "DELETE FROM tbl_scan_log WHERE scan_instance_id = ?"- qvars = [instanceId]- try:- self.dbh.execute(qry1, qvars)- self.dbh.execute(qry2, qvars)- self.dbh.execute(qry3, qvars)- self.dbh.execute(qry4, qvars)- self.conn.commit()- except sqlite3.Error as e:- self.sf.error("SQL error encountered when deleting scan: " + e.args[0])-- # Set the false positive flag for a result- def scanResultsUpdateFP(self, instanceId, resultHashes, fpFlag):- for resultHash in resultHashes:- qry = "UPDATE tbl_scan_results SET false_positive = ? WHERE \- scan_instance_id = ? AND hash = ?"- qvars = [fpFlag, instanceId, resultHash]- try:- self.dbh.execute(qry, qvars)- except sqlite3.Error as e:- self.sf.error("SQL error encountered when updating F/P: " + e.args[0], False)- return False-- self.conn.commit()- return True-- # Store the default configuration- def configSet(self, optMap=dict()):- qry = "REPLACE INTO tbl_config (scope, opt, val) VALUES (?, ?, ?)"- for opt in optMap.keys():- # Module option- if ":" in opt:- parts = opt.split(':')- qvals = [parts[0], parts[1], optMap[opt]]- else:- # Global option- qvals = ["GLOBAL", opt, optMap[opt]]-- try:- self.dbh.execute(qry, qvals)- except sqlite3.Error as e:- self.sf.error("SQL error encountered when storing config, aborting: " + e.args[0])-- self.conn.commit()-- # Retreive the config from the database- def configGet(self):- qry = "SELECT scope, opt, val FROM tbl_config"- try:- retval = dict()- self.dbh.execute(qry)- for [scope, opt, val] in self.dbh.fetchall():- if scope == "GLOBAL":- retval[opt] = val- else:- retval[scope + ":" + opt] = val-- return retval- except sqlite3.Error as e:- self.sf.error("SQL error encountered when fetching configuration: " + e.args[0])-- # Reset the config to default (clear it from the DB and let the hard-coded- # settings in the code take effect.)- def configClear(self):- qry = "DELETE from tbl_config"- try:- self.dbh.execute(qry)- self.conn.commit()- except sqlite3.Error as e:- self.sf.error("Unable to clear configuration from the database: " + e.args[0])-- # Store a configuration value for a scan- def scanConfigSet(self, id, optMap=dict()):- qry = "REPLACE INTO tbl_scan_config \- (scan_instance_id, component, opt, val) VALUES (?, ?, ?, ?)"-- for opt in optMap.keys():- # Module option- if ":" in opt:- parts = opt.split(':')- qvals = [id, parts[0], parts[1], optMap[opt]]- else:- # Global option- qvals = [id, "GLOBAL", opt, optMap[opt]]-- try:- self.dbh.execute(qry, qvals)- except sqlite3.Error as e:- self.sf.error("SQL error encountered when storing config, aborting: " + e.args[0])-- self.conn.commit()-- # Retreive configuration data for a scan component- def scanConfigGet(self, instanceId):- qry = "SELECT component, opt, val FROM tbl_scan_config \- WHERE scan_instance_id = ? ORDER BY component, opt"- qvars = [instanceId]- try:- retval = dict()- self.dbh.execute(qry, qvars)- for [component, opt, val] in self.dbh.fetchall():- if component == "GLOBAL":- retval[opt] = val- else:- retval[component + ":" + opt] = val- return retval- except sqlite3.Error as e:- self.sf.error("SQL error encountered when fetching configuration: " + e.args[0])-- # Store an event- # eventData is a SpiderFootEvent object with the following variables:- # - eventType: the event, e.g. URL_FORM, RAW_DATA, etc.- # - generated: time the event occurred- # - confidence: how sure are we of this data's validity, 0-100- # - visibility: how 'visible' was this data, 0-100- # - risk: how much risk does this data represent, 0-100- # - module: module that generated the event- # - data: the actual data, i.e. a URL, port number, webpage content, etc.- # - sourceEventHash: hash of the event that triggered this event- # And getHash() will return the event hash.- def scanEventStore(self, instanceId, sfEvent, truncateSize=0):- storeData = ''-- if type(sfEvent.data) is not unicode:- # If sfEvent.data is a dict or list, convert it to a string first, as- # those types do not have a unicode converter.- if type(sfEvent.data) is str:- storeData = unicode(sfEvent.data, 'utf-8', errors='replace')- else:- try:- storeData = unicode(str(sfEvent.data), 'utf-8', errors='replace')- except BaseException as e:- self.sf.fatal("Unhandled type detected: " + str(type(sfEvent.data)))- else:- storeData = sfEvent.data-- if truncateSize > 0:- storeData = storeData[0:truncateSize]-- if sfEvent.sourceEventHash in ["", None]:- self.sf.fatal("UNABLE TO CREATE RECORD WITH EMPTY SOURCE EVENT HASH!")-- qry = "INSERT INTO tbl_scan_results \- (scan_instance_id, hash, type, generated, confidence, \- visibility, risk, module, data, source_event_hash) \- VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"- qvals = [instanceId, sfEvent.getHash(), sfEvent.eventType, sfEvent.generated,- sfEvent.confidence, sfEvent.visibility, sfEvent.risk,- sfEvent.module, storeData, sfEvent.sourceEventHash]-- #print("STORING: " + str(qvals))-- try:- self.dbh.execute(qry, qvals)- self.conn.commit()- return None- except sqlite3.Error as e:- self.sf.fatal("SQL error encountered when storing event data (" + str(self.dbh) + ": " + e.args[0])-- # List of all previously run scans- def scanInstanceList(self):- # SQLite doesn't support OUTER JOINs, so we need a work-around that- # does a UNION of scans with results and scans without results to- # get a complete listing.- qry = "SELECT i.guid, i.name, i.seed_target, ROUND(i.created/1000), \- ROUND(i.started)/1000 as started, ROUND(i.ended)/1000, i.status, COUNT(r.type) \- FROM tbl_scan_instance i, tbl_scan_results r WHERE i.guid = r.scan_instance_id \- AND r.type <> 'ROOT' GROUP BY i.guid \- UNION ALL \- SELECT i.guid, i.name, i.seed_target, ROUND(i.created/1000), \- ROUND(i.started)/1000 as started, ROUND(i.ended)/1000, i.status, '0' \- FROM tbl_scan_instance i WHERE i.guid NOT IN ( \- SELECT distinct scan_instance_id FROM tbl_scan_results WHERE type <> 'ROOT') \- ORDER BY started DESC"- try:- self.dbh.execute(qry)- return self.dbh.fetchall()- except sqlite3.Error as e:- self.sf.error("SQL error encountered when fetching scan list: " + e.args[0])-- # History of data from the scan- def scanResultHistory(self, instanceId):- qry = "SELECT STRFTIME('%H:%M %w', generated, 'unixepoch') AS hourmin, \- type, COUNT(*) FROM tbl_scan_results \- WHERE scan_instance_id = ? GROUP BY hourmin, type"- qvars = [instanceId]- try:- self.dbh.execute(qry, qvars)- return self.dbh.fetchall()- except sqlite3.Error as e:- self.sf.error("SQL error encountered when fetching scan history: " + e.args[0])--- # Get the source IDs, types and data for a set of IDs- def scanElementSourcesDirect(self, instanceId, elementIdList):- # the output of this needs to be aligned with scanResultEvent,- # as other functions call both expecting the same output.- qry = "SELECT ROUND(c.generated) AS generated, c.data, \- s.data as 'source_data', \- c.module, c.type, c.confidence, c.visibility, c.risk, c.hash, \- c.source_event_hash, t.event_descr, t.event_type, s.scan_instance_id, \- c.false_positive as 'fp', s.false_positive as 'parent_fp' \- FROM tbl_scan_results c, tbl_scan_results s, tbl_event_types t \- WHERE c.scan_instance_id = ? AND c.source_event_hash = s.hash AND \- s.scan_instance_id = c.scan_instance_id AND \- t.event = c.type AND c.hash in ("- qvars = [instanceId]-- for hashId in elementIdList:- qry = qry + "'" + hashId + "',"- qry += "'')"-- try:- self.dbh.execute(qry, qvars)- return self.dbh.fetchall()- except sqlite3.Error as e:- self.sf.error("SQL error encountered when getting source element IDs: " + e.args[0])-- # Get the child IDs, types and data for a set of IDs- def scanElementChildrenDirect(self, instanceId, elementIdList):- # the output of this needs to be aligned with scanResultEvent,- # as other functions call both expecting the same output.- qry = "SELECT ROUND(c.generated) AS generated, c.data, \- s.data as 'source_data', \- c.module, c.type, c.confidence, c.visibility, c.risk, c.hash, \- c.source_event_hash, t.event_descr, t.event_type, s.scan_instance_id, \- c.false_positive as 'fp', s.false_positive as 'parent_fp' \- FROM tbl_scan_results c, tbl_scan_results s, tbl_event_types t \- WHERE c.scan_instance_id = ? AND c.source_event_hash = s.hash AND \- s.scan_instance_id = c.scan_instance_id AND \- t.event = c.type AND s.hash in ("- qvars = [instanceId]-- for hashId in elementIdList:- qry = qry + "'" + hashId + "',"- qry += "'')"-- try:- self.dbh.execute(qry, qvars)- return self.dbh.fetchall()- except sqlite3.Error as e:- self.sf.error("SQL error encountered when getting child element IDs: " + e.args[0])-- # Get the full set of upstream IDs which are parents to the- # supplied set of IDs.- # Data has to be in the format of output from scanElementSourcesDirect- # and produce output in the same format.- def scanElementSourcesAll(self, instanceId, childData):- # Get the first round of source IDs for the leafs- keepGoing = True- nextIds = list()- datamap = dict()- pc = dict()-- for row in childData:- # these must be unique values!- parentId = row[9]- childId = row[8]- datamap[childId] = row-- if parentId in pc:- if childId not in pc[parentId]:- pc[parentId].append(childId)- else:- pc[parentId] = [childId]-- # parents of the leaf set- if parentId not in nextIds:- nextIds.append(parentId)-- while keepGoing:- parentSet = self.scanElementSourcesDirect(instanceId, nextIds)- nextIds = list()- keepGoing = False-- for row in parentSet:- parentId = row[9]- childId = row[8]- datamap[childId] = row- #print(childId + " = " + str(row))-- if parentId in pc:- if childId not in pc[parentId]:- pc[parentId].append(childId)- else:- pc[parentId] = [childId]- if parentId not in nextIds:- nextIds.append(parentId)-- # Prevent us from looping at root- if parentId != "ROOT":- keepGoing = True-- datamap[parentId] = row- return [datamap, pc]-- # Get the full set of downstream IDs which are children of the- # supplied set of IDs- # NOTE FOR NOW THE BEHAVIOR IS NOT THE SAME AS THE scanElementParent*- # FUNCTIONS - THIS ONLY RETURNS IDS!!- def scanElementChildrenAll(self, instanceId, parentIds):- datamap = list()- keepGoing = True- nextIds = list()-- nextSet = self.scanElementChildrenDirect(instanceId, parentIds)- for row in nextSet:- datamap.append(row[8])-- for row in nextSet:- if row[8] not in nextIds:- nextIds.append(row[8])-- while keepGoing:- nextSet = self.scanElementChildrenDirect(instanceId, nextIds)- if nextSet == None or len(nextSet) == 0:- keepGoing = False- break-- for row in nextSet:- datamap.append(row[8])- nextIds = list()- nextIds.append(row[8])-- return datamap+# -*- coding: utf-8 -*-+# -------------------------------------------------------------------------------+# Name: sfdb+# Purpose: Common functions for working with the database back-end.+#+# Author: Steve Micallef <[email protected]>+#+# Created: 15/05/2012+# Copyright: (c) Steve Micallef 2012+# Licence: GPL+# -------------------------------------------------------------------------------++import sqlite3+import re+import time+from sflib import SpiderFoot++# SQLite doesn't support regex queries, so we create+# a custom function to do so..+def __dbregex__(qry, data):+ try:+ rx = re.compile(qry, re.IGNORECASE|re.DOTALL)+ ret = rx.match(data)+ except BaseException as e:+ return False+ return ret is not None+++class SpiderFootDb:+ sf = None+ dbh = None+ conn = None++ # Queries for creating the SpiderFoot database+ createSchemaQueries = [+ "PRAGMA journal_mode=WAL",+ "CREATE TABLE tbl_event_types ( \+ event VARCHAR NOT NULL PRIMARY KEY, \+ event_descr VARCHAR NOT NULL, \+ event_raw INT NOT NULL DEFAULT 0, \+ event_type VARCHAR NOT NULL \+ )",+ "CREATE TABLE tbl_config ( \+ scope VARCHAR NOT NULL, \+ opt VARCHAR NOT NULL, \+ val VARCHAR NOT NULL, \+ PRIMARY KEY (scope, opt) \+ )",+ "CREATE TABLE tbl_scan_instance ( \+ guid VARCHAR NOT NULL PRIMARY KEY, \+ name VARCHAR NOT NULL, \+ seed_target VARCHAR NOT NULL, \+ created INT DEFAULT 0, \+ started INT DEFAULT 0, \+ ended INT DEFAULT 0, \+ status VARCHAR NOT NULL \+ )",+ "CREATE TABLE tbl_scan_log ( \+ scan_instance_id VARCHAR NOT NULL REFERENCES tbl_scan_instance(guid), \+ generated INT NOT NULL, \+ component VARCHAR, \+ type VARCHAR NOT NULL, \+ message VARCHAR \+ )",+ "CREATE TABLE tbl_scan_config ( \+ scan_instance_id VARCHAR NOT NULL REFERENCES tbl_scan_instance(guid), \+ component VARCHAR NOT NULL, \+ opt VARCHAR NOT NULL, \+ val VARCHAR NOT NULL \+ )",+ "CREATE TABLE tbl_scan_results ( \+ scan_instance_id VARCHAR NOT NULL REFERENCES tbl_scan_instance(guid), \+ hash VARCHAR NOT NULL, \+ type VARCHAR NOT NULL REFERENCES tbl_event_types(event), \+ generated INT NOT NULL, \+ confidence INT NOT NULL DEFAULT 100, \+ visibility INT NOT NULL DEFAULT 100, \+ risk INT NOT NULL DEFAULT 0, \+ module VARCHAR NOT NULL, \+ data VARCHAR, \+ false_positive INT NOT NULL DEFAULT 0, \+ source_event_hash VARCHAR DEFAULT 'ROOT' \+ )",+ "CREATE INDEX idx_scan_results_id ON tbl_scan_results (scan_instance_id)",+ "CREATE INDEX idx_scan_results_type ON tbl_scan_results (scan_instance_id, type)",+ "CREATE INDEX idx_scan_results_hash ON tbl_scan_results (scan_instance_id, hash)",+ "CREATE INDEX idx_scan_results_srchash ON tbl_scan_results (scan_instance_id, source_event_hash)",+ "CREATE INDEX idx_scan_logs ON tbl_scan_log (scan_instance_id)"++ ]++ createTypeQueries = [+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('ROOT', 'Internal SpiderFoot Root event', 1, 'INTERNAL')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('ACCOUNT_EXTERNAL_OWNED', 'Account on External Site', 0, 'ENTITY')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('ACCOUNT_EXTERNAL_OWNED_COMPROMISED', 'Hacked Account on External Site', 0, 'DESCRIPTOR')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('ACCOUNT_EXTERNAL_USER_SHARED_COMPROMISED', 'Hacked User Account on External Site', 0, 'DESCRIPTOR')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('AFFILIATE_EMAILADDR', 'Affiliate - Email Address', 0, 'ENTITY')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('AFFILIATE_INTERNET_NAME', 'Affiliate - Internet Name', 0, 'ENTITY')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('AFFILIATE_IPADDR', 'Affiliate - IP Address', 0, 'ENTITY')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('AFFILIATE_WEB_CONTENT', 'Affiliate - Web Content', 1, 'DATA')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('AFFILIATE_DOMAIN', 'Affiliate - Domain Name', 0, 'ENTITY')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('AFFILIATE_COMPANY_NAME', 'Affiliate - Company Name', 0, 'ENTITY')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('AFFILIATE_DOMAIN_WHOIS', 'Affiliate - Domain Whois', 1, 'DATA')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('AFFILIATE_DESCRIPTION_CATEGORY', 'Affiliate Description - Category', 0, 'DESCRIPTOR')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('AFFILIATE_DESCRIPTION_ABSTRACT', 'Affiliate Description - Abstract', 0, 'DESCRIPTOR')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('APPSTORE_ENTRY', 'App Store Entry', 0, 'ENTITY')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('CLOUD_STORAGE_BUCKET', 'Cloud Storage Bucket', 0, 'ENTITY')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('CLOUD_STORAGE_BUCKET_OPEN', 'Cloud Storage Bucket Open', 0, 'DESCRIPTOR')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('COMPANY_NAME', 'Company Name', 0, 'ENTITY')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('BASE64_DATA', 'Base64-encoded Data', 1, 'DATA')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('BITCOIN_ADDRESS', 'Bitcoin Address', 0, 'ENTITY')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('BITCOIN_BALANCE', 'Bitcoin Balance', 0, 'DESCRIPTOR')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('BGP_AS_OWNER', 'BGP AS Ownership', 0, 'ENTITY')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('BGP_AS_MEMBER', 'BGP AS Membership', 0, 'ENTITY')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('BGP_AS_PEER', 'BGP AS Peer', 0, 'ENTITY')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('BLACKLISTED_IPADDR', 'Blacklisted IP Address', 0, 'DESCRIPTOR')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('BLACKLISTED_AFFILIATE_IPADDR', 'Blacklisted Affiliate IP Address', 0, 'DESCRIPTOR')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('BLACKLISTED_SUBNET', 'Blacklisted IP on Same Subnet', 0, 'DESCRIPTOR')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('BLACKLISTED_NETBLOCK', 'Blacklisted IP on Owned Netblock', 0, 'DESCRIPTOR')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('CO_HOSTED_SITE', 'Co-Hosted Site', 0, 'ENTITY')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('CO_HOSTED_SITE_DOMAIN', 'Co-Hosted Site - Domain Name', 0, 'ENTITY')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('CO_HOSTED_SITE_DOMAIN_WHOIS', 'Co-Hosted Site - Domain Whois', 1, 'DATA')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('DARKNET_MENTION_URL', 'Darknet Mention URL', 0, 'DESCRIPTOR')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('DARKNET_MENTION_CONTENT', 'Darknet Mention Web Content', 1, 'DATA')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('DATE_HUMAN_DOB', 'Date of Birth', 0, 'ENTITY')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('DEFACED_INTERNET_NAME', 'Defaced', 0, 'DESCRIPTOR')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('DEFACED_IPADDR', 'Defaced IP Address', 0, 'DESCRIPTOR')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('DEFACED_AFFILIATE_INTERNET_NAME', 'Defaced Affiliate', 0, 'DESCRIPTOR')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('DEFACED_COHOST', 'Defaced Co-Hosted Site', 0, 'DESCRIPTOR')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('DEFACED_AFFILIATE_IPADDR', 'Defaced Affiliate IP Address', 0, 'DESCRIPTOR')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('DESCRIPTION_CATEGORY', 'Description - Category', 0, 'DESCRIPTOR')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('DESCRIPTION_ABSTRACT', 'Description - Abstract', 0, 'DESCRIPTOR')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('DEVICE_TYPE', 'Device Type', 0, 'DESCRIPTOR')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('DNS_TEXT', 'DNS TXT Record', 0, 'DATA')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('DNS_SRV', 'DNS SRV Record', 0, 'DATA')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('DNS_SPF', 'DNS SPF Record', 0, 'DATA')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('DOMAIN_NAME', 'Domain Name', 0, 'ENTITY')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('DOMAIN_NAME_PARENT', 'Domain Name (Parent)', 0, 'ENTITY')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('DOMAIN_REGISTRAR', 'Domain Registrar', 0, 'ENTITY')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('DOMAIN_WHOIS', 'Domain Whois', 1, 'DATA')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('EMAILADDR', 'Email Address', 0, 'ENTITY')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('EMAILADDR_COMPROMISED', 'Hacked Email Address', 0, 'DESCRIPTOR')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('ERROR_MESSAGE', 'Error Message', 0, 'DATA')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('ETHEREUM_ADDRESS', 'Ethereum Address', 0, 'ENTITY')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('GEOINFO', 'Physical Location', 0, 'DESCRIPTOR')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('HTTP_CODE', 'HTTP Status Code', 0, 'DATA')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('HUMAN_NAME', 'Human Name', 0, 'ENTITY')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('INTERESTING_FILE', 'Interesting File', 0, 'DESCRIPTOR')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('INTERESTING_FILE_HISTORIC', 'Historic Interesting File', 0, 'DESCRIPTOR')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('JUNK_FILE', 'Junk File', 0, 'DESCRIPTOR')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('INTERNET_NAME', 'Internet Name', 0, 'ENTITY')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('INTERNET_NAME_UNRESOLVED', 'Internet Name - Unresolved', 0, 'ENTITY')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('IP_ADDRESS', 'IP Address', 0, 'ENTITY')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('IPV6_ADDRESS', 'IPv6 Address', 0, 'ENTITY')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('LINKED_URL_INTERNAL', 'Linked URL - Internal', 0, 'SUBENTITY')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('LINKED_URL_EXTERNAL', 'Linked URL - External', 0, 'SUBENTITY')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('MALICIOUS_ASN', 'Malicious AS', 0, 'DESCRIPTOR')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('MALICIOUS_IPADDR', 'Malicious IP Address', 0, 'DESCRIPTOR')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('MALICIOUS_COHOST', 'Malicious Co-Hosted Site', 0, 'DESCRIPTOR')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('MALICIOUS_EMAILADDR', 'Malicious E-mail Address', 0, 'DESCRIPTOR')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('MALICIOUS_INTERNET_NAME', 'Malicious Internet Name', 0, 'DESCRIPTOR')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('MALICIOUS_AFFILIATE_INTERNET_NAME', 'Malicious Affiliate', 0, 'DESCRIPTOR')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('MALICIOUS_AFFILIATE_IPADDR', 'Malicious Affiliate IP Address', 0, 'DESCRIPTOR')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('MALICIOUS_NETBLOCK', 'Malicious IP on Owned Netblock', 0, 'DESCRIPTOR')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('MALICIOUS_PHONE_NUMBER', 'Malicious Phone Number', 0, 'DESCRIPTOR')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('MALICIOUS_SUBNET', 'Malicious IP on Same Subnet', 0, 'DESCRIPTOR')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('NETBLOCK_OWNER', 'Netblock Ownership', 0, 'ENTITY')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('NETBLOCK_MEMBER', 'Netblock Membership', 0, 'ENTITY')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('NETBLOCK_WHOIS', 'Netblock Whois', 1, 'DATA')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('OPERATING_SYSTEM', 'Operating System', 0, 'DESCRIPTOR')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('LEAKSITE_URL', 'Leak Site URL', 0, 'ENTITY')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('LEAKSITE_CONTENT', 'Leak Site Content', 1, 'DATA')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('PHONE_NUMBER', 'Phone Number', 0, 'ENTITY')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('PHYSICAL_ADDRESS', 'Physical Address', 0, 'ENTITY')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('PHYSICAL_COORDINATES', 'Physical Coordinates', 0, 'ENTITY')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('PGP_KEY', 'PGP Public Key', 0, 'DATA')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('PROVIDER_DNS', 'Name Server (DNS ''NS'' Records)', 0, 'ENTITY')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('PROVIDER_JAVASCRIPT', 'Externally Hosted Javascript', 0, 'ENTITY')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('PROVIDER_MAIL', 'Email Gateway (DNS ''MX'' Records)', 0, 'ENTITY')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('PROVIDER_HOSTING', 'Hosting Provider', 0, 'ENTITY')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('PROVIDER_TELCO', 'Telecommunications Provider', 0, 'ENTITY')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('PUBLIC_CODE_REPO', 'Public Code Repository', 0, 'ENTITY')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('RAW_RIR_DATA', 'Raw Data from RIRs/APIs', 1, 'DATA')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('RAW_DNS_RECORDS', 'Raw DNS Records', 1, 'DATA')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('RAW_FILE_META_DATA', 'Raw File Meta Data', 1, 'DATA')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('SEARCH_ENGINE_WEB_CONTENT', 'Search Engine''s Web Content', 1, 'DATA')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('SOCIAL_MEDIA', 'Social Media Presence', 0, 'ENTITY')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('SIMILARDOMAIN', 'Similar Domain', 0, 'ENTITY')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('SIMILARDOMAIN_WHOIS', 'Similar Domain - Whois', 1, 'DATA')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('SOFTWARE_USED', 'Software Used', 0, 'SUBENTITY')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('SSL_CERTIFICATE_RAW', 'SSL Certificate - Raw Data', 1, 'DATA')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('SSL_CERTIFICATE_ISSUED', 'SSL Certificate - Issued to', 0, 'ENTITY')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('SSL_CERTIFICATE_ISSUER', 'SSL Certificate - Issued by', 0, 'ENTITY')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('SSL_CERTIFICATE_MISMATCH', 'SSL Certificate Host Mismatch', 0, 'DESCRIPTOR')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('SSL_CERTIFICATE_EXPIRED', 'SSL Certificate Expired', 0, 'DESCRIPTOR')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('SSL_CERTIFICATE_EXPIRING', 'SSL Certificate Expiring', 0, 'DESCRIPTOR')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('TARGET_WEB_CONTENT', 'Web Content', 1, 'DATA')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('TARGET_WEB_CONTENT_TYPE', 'Web Content Type', 0, 'DESCRIPTOR')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('TARGET_WEB_COOKIE', 'Cookies', 0, 'DATA')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('TCP_PORT_OPEN', 'Open TCP Port', 0, 'SUBENTITY')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('TCP_PORT_OPEN_BANNER', 'Open TCP Port Banner', 0, 'DATA')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('UDP_PORT_OPEN', 'Open UDP Port', 0, 'SUBENTITY')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('UDP_PORT_OPEN_INFO', 'Open UDP Port Information', 0, 'DATA')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('URL_ADBLOCKED_EXTERNAL', 'URL (AdBlocked External)', 0, 'DESCRIPTOR')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('URL_ADBLOCKED_INTERNAL', 'URL (AdBlocked Internal)', 0, 'DESCRIPTOR')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('URL_FORM', 'URL (Form)', 0, 'DESCRIPTOR')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('URL_FLASH', 'URL (Uses Flash)', 0, 'DESCRIPTOR')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('URL_JAVASCRIPT', 'URL (Uses Javascript)', 0, 'DESCRIPTOR')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('URL_WEB_FRAMEWORK', 'URL (Uses a Web Framework)', 0, 'DESCRIPTOR')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('URL_JAVA_APPLET', 'URL (Uses Java Applet)', 0, 'DESCRIPTOR')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('URL_STATIC', 'URL (Purely Static)', 0, 'DESCRIPTOR')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('URL_PASSWORD', 'URL (Accepts Passwords)', 0, 'DESCRIPTOR')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('URL_UPLOAD', 'URL (Accepts Uploads)', 0, 'DESCRIPTOR')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('URL_FORM_HISTORIC', 'Historic URL (Form)', 0, 'DESCRIPTOR')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('URL_FLASH_HISTORIC', 'Historic URL (Uses Flash)', 0, 'DESCRIPTOR')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('URL_JAVASCRIPT_HISTORIC', 'Historic URL (Uses Javascript)', 0, 'DESCRIPTOR')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('URL_WEB_FRAMEWORK_HISTORIC', 'Historic URL (Uses a Web Framework)', 0, 'DESCRIPTOR')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('URL_JAVA_APPLET_HISTORIC', 'Historic URL (Uses Java Applet)', 0, 'DESCRIPTOR')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('URL_STATIC_HISTORIC', 'Historic URL (Purely Static)', 0, 'DESCRIPTOR')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('URL_PASSWORD_HISTORIC', 'Historic URL (Accepts Passwords)', 0, 'DESCRIPTOR')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('URL_UPLOAD_HISTORIC', 'Historic URL (Accepts Uploads)', 0, 'DESCRIPTOR')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('USERNAME', 'Username', 0, 'ENTITY')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('VULNERABILITY', 'Vulnerability in Public Domain', 0, 'DESCRIPTOR')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('WEB_ANALYTICS_ID', 'Web Analytics', 0, 'ENTITY')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('WEBSERVER_BANNER', 'Web Server', 0, 'DATA')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('WEBSERVER_HTTPHEADERS', 'HTTP Headers', 1, 'DATA')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('WEBSERVER_STRANGEHEADER', 'Non-Standard HTTP Header', 0, 'DATA')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('WEBSERVER_TECHNOLOGY', 'Web Technology', 0, 'DESCRIPTOR')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('WIFI_ACCESS_POINT', 'WiFi Access Point Nearby', 0, 'ENTITY')",+ "INSERT INTO tbl_event_types (event, event_descr, event_raw, event_type) VALUES ('WIKIPEDIA_PAGE_EDIT', 'Wikipedia Page Edit', 0, 'DESCRIPTOR')"+ ]++ def __init__(self, opts, init=False):+ self.sf = SpiderFoot(opts)++ # connect() will create the database file if it doesn't exist, but+ # at least we can use this opportunity to ensure we have permissions to+ # read and write to such a file.+ dbh = sqlite3.connect(self.sf.myPath() + "/" + opts['__database'], timeout=10)+ if dbh is None:+ self.sf.fatal("Could not connect to internal database, and couldn't create " + opts['__database'])+ dbh.text_factory = str++ self.conn = dbh+ self.dbh = dbh.cursor()++ # Now we actually check to ensure the database file has the schema set+ # up correctly.+ try:+ self.dbh.execute('SELECT COUNT(*) FROM tbl_scan_config')+ self.conn.create_function("REGEXP", 2, __dbregex__)+ except sqlite3.Error:+ # .. If not set up, we set it up.+ try:+ self.create()+ init = True+ except BaseException as e:+ self.sf.error("Tried to set up the SpiderFoot database schema, but failed: " + e.args[0])+ return++ if init:+ print("Attempting to verify database and update if necessary...")+ for qry in self.createTypeQueries:+ try:+ self.dbh.execute(qry)+ self.conn.commit()+ except BaseException as e:+ continue+ self.conn.commit()+ #self.conn.close()++ #+ # Back-end database operations+ #++ # Create the back-end schema+ def create(self):+ try:+ for qry in self.createSchemaQueries:+ self.dbh.execute(qry)+ self.conn.commit()+ for qry in self.createTypeQueries:+ self.dbh.execute(qry)+ self.conn.commit()+ except sqlite3.Error as e:+ raise BaseException("SQL error encountered when setting up database: " + e.args[0])++ # Close the database handle+ def close(self):+ self.dbh.close()++ # Search results+ # criteria is search criteria such as:+ # - scan_id (search within a scan, if omitted search all)+ # - type (search a specific type, if omitted search all)+ # - value (search values for a specific string, if omitted search all)+ # - regex (search values for a regular expression)+ # ** at least two criteria must be set **+ def search(self, criteria, filterFp=False):+ if criteria.values().count(None) == 3:+ return False++ qvars = list()+ qry = "SELECT ROUND(c.generated) AS generated, c.data, \+ s.data as 'source_data', \+ c.module, c.type, c.confidence, c.visibility, c.risk, c.hash, \+ c.source_event_hash, t.event_descr, t.event_type, c.scan_instance_id, \+ c.false_positive as 'fp', s.false_positive as 'parent_fp' \+ FROM tbl_scan_results c, tbl_scan_results s, tbl_event_types t \+ WHERE s.scan_instance_id = c.scan_instance_id AND \+ t.event = c.type AND c.source_event_hash = s.hash "++ if filterFp:+ qry += " AND c.false_positive <> 1 "++ if criteria.get('scan_id') is not None:+ qry += "AND c.scan_instance_id = ? "+ qvars.append(criteria['scan_id'])++ if criteria.get('type') is not None:+ qry += " AND c.type = ? "+ qvars.append(criteria['type'])++ if criteria.get('value') is not None:+ qry += " AND (c.data LIKE ? OR s.data LIKE ?) "+ qvars.append(criteria['value'])+ qvars.append(criteria['value'])++ if criteria.get('regex') is not None:+ qry += " AND (c.data REGEXP ? OR s.data REGEXP ?) "+ qvars.append(criteria['regex'])+ qvars.append(criteria['regex'])++ qry += " ORDER BY c.data"++ try:+ #print(qry)+ #print(str(qvars))+ self.dbh.execute(qry, qvars)+ return self.dbh.fetchall()+ except sqlite3.Error as e:+ self.sf.error("SQL error encountered when fetching search results: " + e.args[0])++ # Get event types+ def eventTypes(self):+ qry = "SELECT event_descr, event, event_raw, event_type FROM tbl_event_types"+ try:+ self.dbh.execute(qry)+ return self.dbh.fetchall()+ except sqlite3.Error as e:+ self.sf.error("SQL error encountered when retreiving event types:" + e.args[0])++ # Log an event to the database+ def scanLogEvent(self, instanceId, classification, message, component=None):+ if component is None:+ component = "SpiderFoot"++ qry = "INSERT INTO tbl_scan_log \+ (scan_instance_id, generated, component, type, message) \+ VALUES (?, ?, ?, ?, ?)"+ try:+ self.dbh.execute(qry, (+ instanceId, time.time() * 1000, component, classification, message+ ))+ self.conn.commit()+ except sqlite3.Error as e:+ if "locked" in e.args[0]:+ # TODO: Do something smarter here to handle locked databases+ self.sf.fatal("Unable to log event in DB due to lock: " + e.args[0])+ else:+ self.sf.fatal("Unable to log event in DB: " + e.args[0])++ return True++ # Store a scan instance+ def scanInstanceCreate(self, instanceId, scanName, scanTarget):+ qry = "INSERT INTO tbl_scan_instance \+ (guid, name, seed_target, created, status) \+ VALUES (?, ?, ?, ?, ?)"+ try:+ self.dbh.execute(qry, (+ instanceId, scanName, scanTarget, time.time() * 1000, 'CREATED'+ ))+ self.conn.commit()+ except sqlite3.Error as e:+ self.sf.fatal("Unable to create instance in DB: " + e.args[0])++ return True++ # Update the start time, end time or status (or all 3) of a scan instance+ def scanInstanceSet(self, instanceId, started=None, ended=None, status=None):+ qvars = list()+ qry = "UPDATE tbl_scan_instance SET "++ if started is not None:+ qry += " started = ?,"+ qvars.append(started)++ if ended is not None:+ qry += " ended = ?,"+ qvars.append(ended)++ if status is not None:+ qry += " status = ?,"+ qvars.append(status)++ # guid = guid is a little hack to avoid messing with , placement above+ qry += " guid = guid WHERE guid = ?"+ qvars.append(instanceId)++ try:+ self.dbh.execute(qry, qvars)+ self.conn.commit()+ except sqlite3.Error:+ self.sf.fatal("Unable to set information for the scan instance.")++ # Return info about a scan instance (name, target, created, started,+ # ended, status) - don't need this yet - untested+ def scanInstanceGet(self, instanceId):+ qry = "SELECT name, seed_target, ROUND(created/1000) AS created, \+ ROUND(started/1000) AS started, ROUND(ended/1000) AS ended, status \+ FROM tbl_scan_instance WHERE guid = ?"+ qvars = [instanceId]+ try:+ self.dbh.execute(qry, qvars)+ return self.dbh.fetchone()+ except sqlite3.Error as e:+ self.sf.error("SQL error encountered when retreiving scan instance:" + e.args[0])++ # Obtain a summary of the results per event type+ def scanResultSummary(self, instanceId, by="type"):+ if by == "type":+ qry = "SELECT r.type, e.event_descr, MAX(ROUND(generated)) AS last_in, \+ count(*) AS total, count(DISTINCT r.data) as utotal FROM \+ tbl_scan_results r, tbl_event_types e WHERE e.event = r.type \+ AND r.scan_instance_id = ? GROUP BY r.type ORDER BY e.event_descr"++ if by == "module":+ qry = "SELECT r.module, '', MAX(ROUND(generated)) AS last_in, \+ count(*) AS total, count(DISTINCT r.data) as utotal FROM \+ tbl_scan_results r, tbl_event_types e WHERE e.event = r.type \+ AND r.scan_instance_id = ? GROUP BY r.module ORDER BY r.module DESC"++ if by == "entity":+ qry = "SELECT r.data, e.event_descr, MAX(ROUND(generated)) AS last_in, \+ count(*) AS total, count(DISTINCT r.data) as utotal FROM \+ tbl_scan_results r, tbl_event_types e WHERE e.event = r.type \+ AND r.scan_instance_id = ? \+ AND e.event_type in ('ENTITY') \+ GROUP BY r.data, e.event_descr ORDER BY total DESC limit 50"++ qvars = [instanceId]+ try:+ self.dbh.execute(qry, qvars)+ return self.dbh.fetchall()+ except sqlite3.Error as e:+ self.sf.error("SQL error encountered when fetching result summary: " + e.args[0])++ # Obtain the data for a scan and event type+ def scanResultEvent(self, instanceId, eventType='ALL', filterFp=False):+ qry = "SELECT ROUND(c.generated) AS generated, c.data, \+ s.data as 'source_data', \+ c.module, c.type, c.confidence, c.visibility, c.risk, c.hash, \+ c.source_event_hash, t.event_descr, t.event_type, s.scan_instance_id, \+ c.false_positive as 'fp', s.false_positive as 'parent_fp' \+ FROM tbl_scan_results c, tbl_scan_results s, tbl_event_types t \+ WHERE c.scan_instance_id = ? AND c.source_event_hash = s.hash AND \+ s.scan_instance_id = c.scan_instance_id AND \+ t.event = c.type"++ qvars = [instanceId]++ if eventType != "ALL":+ qry += " AND c.type = ?"+ qvars.append(eventType)++ if filterFp:+ qry += " AND c.false_positive <> 1"++ qry += " ORDER BY c.data"++ try:+ self.dbh.execute(qry, qvars)+ return self.dbh.fetchall()+ except sqlite3.Error as e:+ self.sf.error("SQL error encountered when fetching result events: " + e.args[0])++ # Obtain a unique list of elements+ def scanResultEventUnique(self, instanceId, eventType='ALL', filterFp=False):+ qry = "SELECT DISTINCT data, type, COUNT(*) FROM tbl_scan_results \+ WHERE scan_instance_id = ?"+ qvars = [instanceId]++ if eventType != "ALL":+ qry += " AND type = ?"+ qvars.append(eventType)++ if filterFp:+ qry += " AND false_positive <> 1"++ qry += " GROUP BY type, data ORDER BY COUNT(*)"++ try:+ self.dbh.execute(qry, qvars)+ return self.dbh.fetchall()+ except sqlite3.Error as e:+ self.sf.error("SQL error encountered when fetching unique result events: " + e.args[0])++ # Get scan logs+ def scanLogs(self, instanceId, limit=None, fromRowId=None, reverse=False):+ qry = "SELECT generated AS generated, component, \+ type, message, rowid FROM tbl_scan_log WHERE scan_instance_id = ?"+ if fromRowId:+ qry += " and rowid > ?"++ qry += " ORDER BY generated "+ if reverse:+ qry += "ASC"+ else:+ qry += "DESC"+ qvars = [instanceId]++ if fromRowId:+ qvars.append(fromRowId)++ if limit is not None:+ qry += " LIMIT ?"+ qvars.append(limit)++ try:+ self.dbh.execute(qry, qvars)+ return self.dbh.fetchall()+ except sqlite3.Error as e:+ self.sf.error("SQL error encountered when fetching scan logs: " + e.args[0])++ # Get scan errors+ def scanErrors(self, instanceId, limit=None):+ qry = "SELECT generated AS generated, component, \+ message FROM tbl_scan_log WHERE scan_instance_id = ? \+ AND type = 'ERROR' ORDER BY generated DESC"+ qvars = [instanceId]++ if limit is not None:+ qry += " LIMIT ?"+ qvars.append(limit)++ try:+ self.dbh.execute(qry, qvars)+ return self.dbh.fetchall()+ except sqlite3.Error as e:+ self.sf.error("SQL error encountered when fetching scan errors: " + e.args[0])++ # Delete a scan instance+ def scanInstanceDelete(self, instanceId):+ qry1 = "DELETE FROM tbl_scan_instance WHERE guid = ?"+ qry2 = "DELETE FROM tbl_scan_config WHERE scan_instance_id = ?"+ qry3 = "DELETE FROM tbl_scan_results WHERE scan_instance_id = ?"+ qry4 = "DELETE FROM tbl_scan_log WHERE scan_instance_id = ?"+ qvars = [instanceId]+ try:+ self.dbh.execute(qry1, qvars)+ self.dbh.execute(qry2, qvars)+ self.dbh.execute(qry3, qvars)+ self.dbh.execute(qry4, qvars)+ self.conn.commit()+ except sqlite3.Error as e:+ self.sf.error("SQL error encountered when deleting scan: " + e.args[0])++ # Set the false positive flag for a result+ def scanResultsUpdateFP(self, instanceId, resultHashes, fpFlag):+ for resultHash in resultHashes:+ qry = "UPDATE tbl_scan_results SET false_positive = ? WHERE \+ scan_instance_id = ? AND hash = ?"+ qvars = [fpFlag, instanceId, resultHash]+ try:+ self.dbh.execute(qry, qvars)+ except sqlite3.Error as e:+ self.sf.error("SQL error encountered when updating F/P: " + e.args[0], False)+ return False++ self.conn.commit()+ return True++ # Store the default configuration+ def configSet(self, optMap=dict()):+ qry = "REPLACE INTO tbl_config (scope, opt, val) VALUES (?, ?, ?)"+ for opt in optMap.keys():+ # Module option+ if ":" in opt:+ parts = opt.split(':')+ qvals = [parts[0], parts[1], optMap[opt]]+ else:+ # Global option+ qvals = ["GLOBAL", opt, optMap[opt]]++ try:+ self.dbh.execute(qry, qvals)+ except sqlite3.Error as e:+ self.sf.error("SQL error encountered when storing config, aborting: " + e.args[0])++ self.conn.commit()++ # Retreive the config from the database+ def configGet(self):+ qry = "SELECT scope, opt, val FROM tbl_config"+ try:+ retval = dict()+ self.dbh.execute(qry)+ for [scope, opt, val] in self.dbh.fetchall():+ if scope == "GLOBAL":+ retval[opt] = val+ else:+ retval[scope + ":" + opt] = val++ return retval+ except sqlite3.Error as e:+ self.sf.error("SQL error encountered when fetching configuration: " + e.args[0])++ # Reset the config to default (clear it from the DB and let the hard-coded+ # settings in the code take effect.)+ def configClear(self):+ qry = "DELETE from tbl_config"+ try:+ self.dbh.execute(qry)+ self.conn.commit()+ except sqlite3.Error as e:+ self.sf.error("Unable to clear configuration from the database: " + e.args[0])++ # Store a configuration value for a scan+ def scanConfigSet(self, id, optMap=dict()):+ qry = "REPLACE INTO tbl_scan_config \+ (scan_instance_id, component, opt, val) VALUES (?, ?, ?, ?)"++ for opt in optMap.keys():+ # Module option+ if ":" in opt:+ parts = opt.split(':')+ qvals = [id, parts[0], parts[1], optMap[opt]]+ else:+ # Global option+ qvals = [id, "GLOBAL", opt, optMap[opt]]++ try:+ self.dbh.execute(qry, qvals)+ except sqlite3.Error as e:+ self.sf.error("SQL error encountered when storing config, aborting: " + e.args[0])++ self.conn.commit()++ # Retreive configuration data for a scan component+ def scanConfigGet(self, instanceId):+ qry = "SELECT component, opt, val FROM tbl_scan_config \+ WHERE scan_instance_id = ? ORDER BY component, opt"+ qvars = [instanceId]+ try:+ retval = dict()+ self.dbh.execute(qry, qvars)+ for [component, opt, val] in self.dbh.fetchall():+ if component == "GLOBAL":+ retval[opt] = val+ else:+ retval[component + ":" + opt] = val+ return retval+ except sqlite3.Error as e:+ self.sf.error("SQL error encountered when fetching configuration: " + e.args[0])++ # Store an event+ # eventData is a SpiderFootEvent object with the following variables:+ # - eventType: the event, e.g. URL_FORM, RAW_DATA, etc.+ # - generated: time the event occurred+ # - confidence: how sure are we of this data's validity, 0-100+ # - visibility: how 'visible' was this data, 0-100+ # - risk: how much risk does this data represent, 0-100+ # - module: module that generated the event+ # - data: the actual data, i.e. a URL, port number, webpage content, etc.+ # - sourceEventHash: hash of the event that triggered this event+ # And getHash() will return the event hash.+ def scanEventStore(self, instanceId, sfEvent, truncateSize=0):+ storeData = ''++ if type(sfEvent.data) is not unicode:+ # If sfEvent.data is a dict or list, convert it to a string first, as+ # those types do not have a unicode converter.+ if type(sfEvent.data) is str:+ storeData = unicode(sfEvent.data, 'utf-8', errors='replace')+ else:+ try:+ storeData = unicode(str(sfEvent.data), 'utf-8', errors='replace')+ except BaseException as e:+ self.sf.fatal("Unhandled type detected: " + str(type(sfEvent.data)))+ else:+ storeData = sfEvent.data++ if truncateSize > 0:+ storeData = storeData[0:truncateSize]++ if sfEvent.sourceEventHash in ["", None]:+ self.sf.fatal("UNABLE TO CREATE RECORD WITH EMPTY SOURCE EVENT HASH!")++ qry = "INSERT INTO tbl_scan_results \+ (scan_instance_id, hash, type, generated, confidence, \+ visibility, risk, module, data, source_event_hash) \+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"+ qvals = [instanceId, sfEvent.getHash(), sfEvent.eventType, sfEvent.generated,+ sfEvent.confidence, sfEvent.visibility, sfEvent.risk,+ sfEvent.module, storeData, sfEvent.sourceEventHash]++ #print("STORING: " + str(qvals))++ try:+ self.dbh.execute(qry, qvals)+ self.conn.commit()+ return None+ except sqlite3.Error as e:+ self.sf.fatal("SQL error encountered when storing event data (" + str(self.dbh) + ": " + e.args[0])++ # List of all previously run scans+ def scanInstanceList(self):+ # SQLite doesn't support OUTER JOINs, so we need a work-around that+ # does a UNION of scans with results and scans without results to+ # get a complete listing.+ qry = "SELECT i.guid, i.name, i.seed_target, ROUND(i.created/1000), \+ ROUND(i.started)/1000 as started, ROUND(i.ended)/1000, i.status, COUNT(r.type) \+ FROM tbl_scan_instance i, tbl_scan_results r WHERE i.guid = r.scan_instance_id \+ AND r.type <> 'ROOT' GROUP BY i.guid \+ UNION ALL \+ SELECT i.guid, i.name, i.seed_target, ROUND(i.created/1000), \+ ROUND(i.started)/1000 as started, ROUND(i.ended)/1000, i.status, '0' \+ FROM tbl_scan_instance i WHERE i.guid NOT IN ( \+ SELECT distinct scan_instance_id FROM tbl_scan_results WHERE type <> 'ROOT') \+ ORDER BY started DESC"+ try:+ self.dbh.execute(qry)+ return self.dbh.fetchall()+ except sqlite3.Error as e:+ self.sf.error("SQL error encountered when fetching scan list: " + e.args[0])++ # History of data from the scan+ def scanResultHistory(self, instanceId):+ qry = "SELECT STRFTIME('%H:%M %w', generated, 'unixepoch') AS hourmin, \+ type, COUNT(*) FROM tbl_scan_results \+ WHERE scan_instance_id = ? GROUP BY hourmin, type"+ qvars = [instanceId]+ try:+ self.dbh.execute(qry, qvars)+ return self.dbh.fetchall()+ except sqlite3.Error as e:+ self.sf.error("SQL error encountered when fetching scan history: " + e.args[0])+++ # Get the source IDs, types and data for a set of IDs+ def scanElementSourcesDirect(self, instanceId, elementIdList):+ # the output of this needs to be aligned with scanResultEvent,+ # as other functions call both expecting the same output.+ qry = "SELECT ROUND(c.generated) AS generated, c.data, \+ s.data as 'source_data', \+ c.module, c.type, c.confidence, c.visibility, c.risk, c.hash, \+ c.source_event_hash, t.event_descr, t.event_type, s.scan_instance_id, \+ c.false_positive as 'fp', s.false_positive as 'parent_fp' \+ FROM tbl_scan_results c, tbl_scan_results s, tbl_event_types t \+ WHERE c.scan_instance_id = ? AND c.source_event_hash = s.hash AND \+ s.scan_instance_id = c.scan_instance_id AND \+ t.event = c.type AND c.hash in ("+ qvars = [instanceId]++ for hashId in elementIdList:+ qry = qry + "'" + hashId + "',"+ qry += "'')"++ try:+ self.dbh.execute(qry, qvars)+ return self.dbh.fetchall()+ except sqlite3.Error as e:+ self.sf.error("SQL error encountered when getting source element IDs: " + e.args[0])++ # Get the child IDs, types and data for a set of IDs+ def scanElementChildrenDirect(self, instanceId, elementIdList):+ # the output of this needs to be aligned with scanResultEvent,+ # as other functions call both expecting the same output.+ qry = "SELECT ROUND(c.generated) AS generated, c.data, \+ s.data as 'source_data', \+ c.module, c.type, c.confidence, c.visibility, c.risk, c.hash, \+ c.source_event_hash, t.event_descr, t.event_type, s.scan_instance_id, \+ c.false_positive as 'fp', s.false_positive as 'parent_fp' \+ FROM tbl_scan_results c, tbl_scan_results s, tbl_event_types t \+ WHERE c.scan_instance_id = ? AND c.source_event_hash = s.hash AND \+ s.scan_instance_id = c.scan_instance_id AND \+ t.event = c.type AND s.hash in ("+ qvars = [instanceId]++ for hashId in elementIdList:+ qry = qry + "'" + hashId + "',"+ qry += "'')"++ try:+ self.dbh.execute(qry, qvars)+ return self.dbh.fetchall()+ except sqlite3.Error as e:+ self.sf.error("SQL error encountered when getting child element IDs: " + e.args[0])++ # Get the full set of upstream IDs which are parents to the+ # supplied set of IDs.+ # Data has to be in the format of output from scanElementSourcesDirect+ # and produce output in the same format.+ def scanElementSourcesAll(self, instanceId, childData):+ # Get the first round of source IDs for the leafs+ keepGoing = True+ nextIds = list()+ datamap = dict()+ pc = dict()++ for row in childData:+ # these must be unique values!+ parentId = row[9]+ childId = row[8]+ datamap[childId] = row++ if parentId in pc:+ if childId not in pc[parentId]:+ pc[parentId].append(childId)+ else:+ pc[parentId] = [childId]++ # parents of the leaf set+ if parentId not in nextIds:+ nextIds.append(parentId)++ while keepGoing:+ parentSet = self.scanElementSourcesDirect(instanceId, nextIds)+ nextIds = list()+ keepGoing = False++ for row in parentSet:+ parentId = row[9]+ childId = row[8]+ datamap[childId] = row+ #print(childId + " = " + str(row))++ if parentId in pc:+ if childId not in pc[parentId]:+ pc[parentId].append(childId)+ else:+ pc[parentId] = [childId]+ if parentId not in nextIds:+ nextIds.append(parentId)++ # Prevent us from looping at root+ if parentId != "ROOT":+ keepGoing = True++ datamap[parentId] = row+ return [datamap, pc]++ # Get the full set of downstream IDs which are children of the+ # supplied set of IDs+ # NOTE FOR NOW THE BEHAVIOR IS NOT THE SAME AS THE scanElementParent*+ # FUNCTIONS - THIS ONLY RETURNS IDS!!+ def scanElementChildrenAll(self, instanceId, parentIds):+ datamap = list()+ keepGoing = True+ nextIds = list()++ nextSet = self.scanElementChildrenDirect(instanceId, parentIds)+ for row in nextSet:+ datamap.append(row[8])++ for row in nextSet:+ if row[8] not in nextIds:+ nextIds.append(row[8])++ while keepGoing:+ nextSet = self.scanElementChildrenDirect(instanceId, nextIds)+ if nextSet == None or len(nextSet) == 0:+ keepGoing = False+ break++ for row in nextSet:+ datamap.append(row[8])+ nextIds = list()+ nextIds.append(row[8])++ return datamap