SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.
Python attacksurfacecticybersecurityfootprintinghackinginformation-gatheringinformation-securityinfosecintelligence-gatheringosintosint-frameworkosint-reconnaissanceosint-toolpentestingpythonreconsecurity-toolsthreat-intelligencethreatintel
spiderfoot: store data and cache in user's home directory (#1343)
bcoles31b8cba7f110
diff --git a/sf.py b/sf.pyindex e4ee245d..79a28386 100755--- a/sf.py+++ b/sf.py@@ -60,7 +60,7 @@ def main():'_internettlds': 'https://publicsuffix.org/list/effective_tld_names.dat','_internettlds_cache': 72,'_genericusers': "abuse,admin,billing,compliance,devnull,dns,ftp,hostmaster,inoc,ispfeedback,ispsupport,list-request,list,maildaemon,marketing,noc,no-reply,noreply,null,peering,peering-notify,peering-request,phish,phishing,postmaster,privacy,registrar,registry,root,routing-registry,rr,sales,security,spam,support,sysadmin,tech,undisclosed-recipients,unsubscribe,usenet,uucp,webmaster,www",- '__database': 'spiderfoot.db',+ '__database': f"{SpiderFootHelpers.dataPath()}/spiderfoot.db",'__modules__': None, # List of modules. Will be set after start-up.'_socks1type': '','_socks2addr': '',@@ -452,7 +452,7 @@ def start_web_server(sfWebUiConfig, sfConfig, loggingQueue=None):}secrets = dict()- passwd_file = sf.dataPath() + '/passwd'+ passwd_file = SpiderFootHelpers.dataPath() + '/passwd'if os.path.isfile(passwd_file):if not os.access(passwd_file, os.R_OK):log.error("Could not read passwd file. Permission denied.")@@ -494,8 +494,8 @@ def start_web_server(sfWebUiConfig, sfConfig, loggingQueue=None):log.warning(warn_msg)using_ssl = False- key_path = sf.dataPath() + '/spiderfoot.key'- crt_path = sf.dataPath() + '/spiderfoot.crt'+ key_path = SpiderFootHelpers.dataPath() + '/spiderfoot.key'+ crt_path = SpiderFootHelpers.dataPath() + '/spiderfoot.crt'if os.path.isfile(key_path) and os.path.isfile(crt_path):if not os.access(crt_path, os.R_OK):log.critical(f"Could not read {crt_path} file. Permission denied.")@@ -569,4 +569,13 @@ if __name__ == '__main__':print("SpiderFoot requires -l <ip>:<port> to start the web server. Try --help for guidance.")sys.exit(-1)+ # TODO: remove this after a few releases (added in 3.5 pre-release 2021-09-05)+ from pathlib import Path+ if os.path.exists('spiderfoot.db'):+ print(f"ERROR: spiderfoot.db file exists in {os.path.dirname(__file__)}")+ print("SpiderFoot no longer supports loading the spiderfoot.db database from the application directory.")+ print(f"The database is now loaded from your home directory: {Path.home()}/.spiderfoot/spiderfoot.db")+ print(f"This message will go away once you move or remove spiderfoot.db from {os.path.dirname(__file__)}")+ sys.exit(-1)+main()diff --git a/sflib.py b/sflib.pyindex 14ff7df8..a2816e7c 100644--- a/sflib.py+++ b/sflib.py@@ -29,6 +29,7 @@ import urllib.parseimport urllib.requestfrom copy import deepcopyfrom datetime import datetime+from pathlib import Pathimport cryptographyimport dns.resolver@@ -71,7 +72,6 @@ class SpiderFoot:raise TypeError("options is %s; expected dict()" % type(options))self.opts = deepcopy(options)-self.log = logging.getLogger(f"spiderfoot.{__name__}")# This is ugly but we don't want any fetches to fail - we expect@@ -285,16 +285,6 @@ class SpiderFoot:return os.path.dirname(__file__)- @classmethod- def dataPath(cls) -> str:- """Returns the file system location of SpiderFoot data and configuration files.-- Returns:- str: SpiderFoot file system path- """- path = os.environ.get('SPIDERFOOT_DATA')- return path if path is not None else cls.myPath()-def hashstring(self, string: str) -> str:"""Returns a SHA256 hash of the specified input.@@ -317,7 +307,7 @@ class SpiderFoot:"""path = os.environ.get('SPIDERFOOT_CACHE')if not path:- path = self.myPath() + '/cache'+ path = f"{Path.home()}/.spiderfoot/cache"if not os.path.isdir(path):os.mkdir(path)return pathdiff --git a/spiderfoot/helpers.py b/spiderfoot/helpers.pyindex d8d58e8c..91ce52eb 100644--- a/spiderfoot/helpers.py+++ b/spiderfoot/helpers.py@@ -1,15 +1,32 @@import json+import os+import os.pathimport randomimport reimport uuidimport networkx as nxfrom networkx.readwrite.gexf import GEXFWriter+from pathlib import Pathclass SpiderFootHelpers():"""SpiderFoot helper functions."""+ @staticmethod+ def dataPath() -> str:+ """Returns the file system location of SpiderFoot data and configuration files.++ Returns:+ str: SpiderFoot data file system path+ """+ path = os.environ.get('SPIDERFOOT_DATA')+ if not path:+ path = f"{Path.home()}/.spiderfoot/"+ if not os.path.isdir(path):+ os.mkdir(path)+ return path+@staticmethoddef targetTypeFromString(target: str) -> None:"""Return the scan target seed data type for the specified scan target input.diff --git a/test/conftest.py b/test/conftest.pyindex e9706fe7..4db2d5ff 100644--- a/test/conftest.py+++ b/test/conftest.py@@ -1,4 +1,5 @@import pytest+from spiderfoot import SpiderFootHelpers@pytest.fixture(autouse=True)@@ -13,7 +14,7 @@ def default_options(request):'_internettlds': 'https://publicsuffix.org/list/effective_tld_names.dat','_internettlds_cache': 72,'_genericusers': "abuse,admin,billing,compliance,devnull,dns,ftp,hostmaster,inoc,ispfeedback,ispsupport,list-request,list,maildaemon,marketing,noc,no-reply,noreply,null,peering,peering-notify,peering-request,phish,phishing,postmaster,privacy,registrar,registry,root,routing-registry,rr,sales,security,spam,support,sysadmin,tech,undisclosed-recipients,unsubscribe,usenet,uucp,webmaster,www",- '__database': 'spiderfoot.test.db', # note: test database file+ '__database': f"{SpiderFootHelpers.dataPath()}/spiderfoot.test.db", # note: test database file'__modules__': None, # List of modules. Will be set after start-up.'_socks1type': '','_socks2addr': '',diff --git a/test/integration/test_sfwebui.py b/test/integration/test_sfwebui.pyindex 3bce4762..2e5ff8a9 100644--- a/test/integration/test_sfwebui.py+++ b/test/integration/test_sfwebui.py@@ -5,6 +5,7 @@ import unittestimport cherrypyfrom cherrypy.test import helper+from spiderfoot import SpiderFootHelpersfrom sflib import SpiderFootfrom sfwebui import SpiderFootWebUi@@ -22,7 +23,7 @@ class TestSpiderFootWebUiRoutes(helper.CPWebCase):'_internettlds': 'https://publicsuffix.org/list/effective_tld_names.dat','_internettlds_cache': 72,'_genericusers': "abuse,admin,billing,compliance,devnull,dns,ftp,hostmaster,inoc,ispfeedback,ispsupport,list-request,list,maildaemon,marketing,noc,no-reply,noreply,null,peering,peering-notify,peering-request,phish,phishing,postmaster,privacy,registrar,registry,root,routing-registry,rr,sales,security,spam,support,sysadmin,tech,undisclosed-recipients,unsubscribe,usenet,uucp,webmaster,www",- '__database': 'spiderfoot.test.db', # note: test database file+ '__database': f"{SpiderFootHelpers.dataPath()}/spiderfoot.test.db", # note: test database file'__modules__': None, # List of modules. Will be set after start-up.'_socks1type': '','_socks2addr': '',diff --git a/test/run b/test/runindex 6ba217e8..5e92ed50 100755--- a/test/run+++ b/test/run@@ -5,4 +5,4 @@# Must be run from SpiderFoot root directory; ie:# ./test/run-time python3 -m pytest -n auto --flake8 --dist loadfile --durations=5 --cov-report term --cov=. .+time python3 -m pytest -n auto --flake8 --dist loadfile --durations=5 --cov-report html --cov=. .diff --git a/test/unit/test_spiderfoot.py b/test/unit/test_spiderfoot.pyindex 7f09cb42..094c69da 100644--- a/test/unit/test_spiderfoot.py+++ b/test/unit/test_spiderfoot.py@@ -163,15 +163,6 @@ class TestSpiderFoot(unittest.TestCase):path = sf.myPath()self.assertIsInstance(path, str)- def test_data_path_should_return_a_string(self):- """- Test def dataPath(self)- """- sf = SpiderFoot(dict())-- path = sf.myPath()- self.assertIsInstance(path, str)-def test_hash_string_should_return_a_string(self):"""Test hashstring(self, string)