Skip to content

SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.

spiderfoot: store data and cache in user's home directory (#1343)

bcoles31b8cba7f110
diff --git a/sf.py b/sf.py
index e4ee245d..79a28386 100755
--- a/sf.py
+++ b/sf.py
@@ -60,7 +60,7 @@ def main():
'_internettlds': 'https://publicsuffix.org/list/effective_tld_names.dat',
'_internettlds_cache': 72,
'_genericusers': "abuse,admin,billing,compliance,devnull,dns,ftp,hostmaster,inoc,ispfeedback,ispsupport,list-request,list,maildaemon,marketing,noc,no-reply,noreply,null,peering,peering-notify,peering-request,phish,phishing,postmaster,privacy,registrar,registry,root,routing-registry,rr,sales,security,spam,support,sysadmin,tech,undisclosed-recipients,unsubscribe,usenet,uucp,webmaster,www",
- '__database': 'spiderfoot.db',
+ '__database': f"{SpiderFootHelpers.dataPath()}/spiderfoot.db",
'__modules__': None, # List of modules. Will be set after start-up.
'_socks1type': '',
'_socks2addr': '',
@@ -452,7 +452,7 @@ def start_web_server(sfWebUiConfig, sfConfig, loggingQueue=None):
}
secrets = dict()
- passwd_file = sf.dataPath() + '/passwd'
+ passwd_file = SpiderFootHelpers.dataPath() + '/passwd'
if os.path.isfile(passwd_file):
if not os.access(passwd_file, os.R_OK):
log.error("Could not read passwd file. Permission denied.")
@@ -494,8 +494,8 @@ def start_web_server(sfWebUiConfig, sfConfig, loggingQueue=None):
log.warning(warn_msg)
using_ssl = False
- key_path = sf.dataPath() + '/spiderfoot.key'
- crt_path = sf.dataPath() + '/spiderfoot.crt'
+ key_path = SpiderFootHelpers.dataPath() + '/spiderfoot.key'
+ crt_path = SpiderFootHelpers.dataPath() + '/spiderfoot.crt'
if os.path.isfile(key_path) and os.path.isfile(crt_path):
if not os.access(crt_path, os.R_OK):
log.critical(f"Could not read {crt_path} file. Permission denied.")
@@ -569,4 +569,13 @@ if __name__ == '__main__':
print("SpiderFoot requires -l <ip>:<port> to start the web server. Try --help for guidance.")
sys.exit(-1)
+ # TODO: remove this after a few releases (added in 3.5 pre-release 2021-09-05)
+ from pathlib import Path
+ if os.path.exists('spiderfoot.db'):
+ print(f"ERROR: spiderfoot.db file exists in {os.path.dirname(__file__)}")
+ print("SpiderFoot no longer supports loading the spiderfoot.db database from the application directory.")
+ print(f"The database is now loaded from your home directory: {Path.home()}/.spiderfoot/spiderfoot.db")
+ print(f"This message will go away once you move or remove spiderfoot.db from {os.path.dirname(__file__)}")
+ sys.exit(-1)
+
main()
diff --git a/sflib.py b/sflib.py
index 14ff7df8..a2816e7c 100644
--- a/sflib.py
+++ b/sflib.py
@@ -29,6 +29,7 @@ import urllib.parse
import urllib.request
from copy import deepcopy
from datetime import datetime
+from pathlib import Path
import cryptography
import dns.resolver
@@ -71,7 +72,6 @@ class SpiderFoot:
raise TypeError("options is %s; expected dict()" % type(options))
self.opts = deepcopy(options)
-
self.log = logging.getLogger(f"spiderfoot.{__name__}")
# This is ugly but we don't want any fetches to fail - we expect
@@ -285,16 +285,6 @@ class SpiderFoot:
return os.path.dirname(__file__)
- @classmethod
- def dataPath(cls) -> str:
- """Returns the file system location of SpiderFoot data and configuration files.
-
- Returns:
- str: SpiderFoot file system path
- """
- path = os.environ.get('SPIDERFOOT_DATA')
- return path if path is not None else cls.myPath()
-
def hashstring(self, string: str) -> str:
"""Returns a SHA256 hash of the specified input.
@@ -317,7 +307,7 @@ class SpiderFoot:
"""
path = os.environ.get('SPIDERFOOT_CACHE')
if not path:
- path = self.myPath() + '/cache'
+ path = f"{Path.home()}/.spiderfoot/cache"
if not os.path.isdir(path):
os.mkdir(path)
return path
diff --git a/spiderfoot/helpers.py b/spiderfoot/helpers.py
index d8d58e8c..91ce52eb 100644
--- a/spiderfoot/helpers.py
+++ b/spiderfoot/helpers.py
@@ -1,15 +1,32 @@
import json
+import os
+import os.path
import random
import re
import uuid
import networkx as nx
from networkx.readwrite.gexf import GEXFWriter
+from pathlib import Path
class SpiderFootHelpers():
"""SpiderFoot helper functions."""
+ @staticmethod
+ def dataPath() -> str:
+ """Returns the file system location of SpiderFoot data and configuration files.
+
+ Returns:
+ str: SpiderFoot data file system path
+ """
+ path = os.environ.get('SPIDERFOOT_DATA')
+ if not path:
+ path = f"{Path.home()}/.spiderfoot/"
+ if not os.path.isdir(path):
+ os.mkdir(path)
+ return path
+
@staticmethod
def targetTypeFromString(target: str) -> None:
"""Return the scan target seed data type for the specified scan target input.
diff --git a/test/conftest.py b/test/conftest.py
index e9706fe7..4db2d5ff 100644
--- a/test/conftest.py
+++ b/test/conftest.py
@@ -1,4 +1,5 @@
import pytest
+from spiderfoot import SpiderFootHelpers
@pytest.fixture(autouse=True)
@@ -13,7 +14,7 @@ def default_options(request):
'_internettlds': 'https://publicsuffix.org/list/effective_tld_names.dat',
'_internettlds_cache': 72,
'_genericusers': "abuse,admin,billing,compliance,devnull,dns,ftp,hostmaster,inoc,ispfeedback,ispsupport,list-request,list,maildaemon,marketing,noc,no-reply,noreply,null,peering,peering-notify,peering-request,phish,phishing,postmaster,privacy,registrar,registry,root,routing-registry,rr,sales,security,spam,support,sysadmin,tech,undisclosed-recipients,unsubscribe,usenet,uucp,webmaster,www",
- '__database': 'spiderfoot.test.db', # note: test database file
+ '__database': f"{SpiderFootHelpers.dataPath()}/spiderfoot.test.db", # note: test database file
'__modules__': None, # List of modules. Will be set after start-up.
'_socks1type': '',
'_socks2addr': '',
diff --git a/test/integration/test_sfwebui.py b/test/integration/test_sfwebui.py
index 3bce4762..2e5ff8a9 100644
--- a/test/integration/test_sfwebui.py
+++ b/test/integration/test_sfwebui.py
@@ -5,6 +5,7 @@ import unittest
import cherrypy
from cherrypy.test import helper
+from spiderfoot import SpiderFootHelpers
from sflib import SpiderFoot
from sfwebui import SpiderFootWebUi
@@ -22,7 +23,7 @@ class TestSpiderFootWebUiRoutes(helper.CPWebCase):
'_internettlds': 'https://publicsuffix.org/list/effective_tld_names.dat',
'_internettlds_cache': 72,
'_genericusers': "abuse,admin,billing,compliance,devnull,dns,ftp,hostmaster,inoc,ispfeedback,ispsupport,list-request,list,maildaemon,marketing,noc,no-reply,noreply,null,peering,peering-notify,peering-request,phish,phishing,postmaster,privacy,registrar,registry,root,routing-registry,rr,sales,security,spam,support,sysadmin,tech,undisclosed-recipients,unsubscribe,usenet,uucp,webmaster,www",
- '__database': 'spiderfoot.test.db', # note: test database file
+ '__database': f"{SpiderFootHelpers.dataPath()}/spiderfoot.test.db", # note: test database file
'__modules__': None, # List of modules. Will be set after start-up.
'_socks1type': '',
'_socks2addr': '',
diff --git a/test/run b/test/run
index 6ba217e8..5e92ed50 100755
--- a/test/run
+++ b/test/run
@@ -5,4 +5,4 @@
# Must be run from SpiderFoot root directory; ie:
# ./test/run
-time python3 -m pytest -n auto --flake8 --dist loadfile --durations=5 --cov-report term --cov=. .
+time python3 -m pytest -n auto --flake8 --dist loadfile --durations=5 --cov-report html --cov=. .
diff --git a/test/unit/test_spiderfoot.py b/test/unit/test_spiderfoot.py
index 7f09cb42..094c69da 100644
--- a/test/unit/test_spiderfoot.py
+++ b/test/unit/test_spiderfoot.py
@@ -163,15 +163,6 @@ class TestSpiderFoot(unittest.TestCase):
path = sf.myPath()
self.assertIsInstance(path, str)
- def test_data_path_should_return_a_string(self):
- """
- Test def dataPath(self)
- """
- sf = SpiderFoot(dict())
-
- path = sf.myPath()
- self.assertIsInstance(path, str)
-
def test_hash_string_should_return_a_string(self):
"""
Test hashstring(self, string)