SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.
Python attacksurfacecticybersecurityfootprintinghackinginformation-gatheringinformation-securityinfosecintelligence-gatheringosintosint-frameworkosint-reconnaissanceosint-toolpentestingpythonreconsecurity-toolsthreat-intelligencethreatintel
Docker updates, fixes (#1386)
TheTechromancer45f87b4e4f85
diff --git a/Dockerfile b/Dockerfileindex 7d715b64..5e316dba 100644--- a/Dockerfile+++ b/Dockerfile@@ -51,10 +51,11 @@ RUN pip3 install -r "$REQUIREMENTS"FROM alpine:3.13.0WORKDIR /home/spiderfoot-ENV SPIDERFOOT_LOGS /home/spiderfoot/log-# Place database and configs outside installation directory+# Place database and logs outside installation directoryENV SPIDERFOOT_DATA /var/lib/spiderfoot+ENV SPIDERFOOT_LOGS /var/lib/spiderfoot/log+ENV SPIDERFOOT_CACHE /var/lib/spiderfoot/cache# Run everything as one command so that only one layer is createdRUN apk --update --no-cache add python3 musl openssl libxslt tinyxml libxml2 jpeg zlib openjpeg \@@ -64,10 +65,12 @@ RUN apk --update --no-cache add python3 musl openssl libxslt tinyxml libxml2 jpe&& rm -rf /var/cache/apk/* \&& rm -rf /lib/apk/db \&& rm -rf /root/.cache \- && mkdir $SPIDERFOOT_DATA \- && mkdir $SPIDERFOOT_LOGS \+ && mkdir -p $SPIDERFOOT_DATA || true \+ && mkdir -p $SPIDERFOOT_LOGS || true \+ && mkdir -p $SPIDERFOOT_CACHE || true \+ && chown spiderfoot:spiderfoot $SPIDERFOOT_DATA \&& chown spiderfoot:spiderfoot $SPIDERFOOT_LOGS \- && chown spiderfoot:spiderfoot $SPIDERFOOT_DATA+ && chown spiderfoot:spiderfoot $SPIDERFOOT_CACHECOPY . .COPY --from=build /opt/venv /opt/venvdiff --git a/Dockerfile.full b/Dockerfile.fullindex 2c934751..efa16b07 100644--- a/Dockerfile.full+++ b/Dockerfile.full@@ -16,7 +16,7 @@ RUN apt-get -y update && apt-get -y install nmapRUN mkdir /tools || trueWORKDIR /toolsRUN git clone --depth=1 https://github.com/blechschmidt/massdns.git \- && cd massdns && make && make install && cd /tools && rm -r massdns+ && cd massdns && make && make install && cd /tools && rm -r massdns# Install Golang toolsRUN apt-get -y update && apt-get -y install golang@@ -30,54 +30,60 @@ RUN GO111MODULE=on go get -v github.com/projectdiscovery/shuffledns/cmd/shuffledRUN apt-get -y update && apt-get -y install ruby ruby-dev bundler# WhatWebRUN git clone https://github.com/urbanadventurer/WhatWeb \- && gem install rchardet mongo json && cd /tools/WhatWeb \- && bundle install && cd /tools+ && gem install rchardet mongo json && cd /tools/WhatWeb \+ && bundle install && cd /toolsRUN groupadd spiderfoot \&& useradd -m -g spiderfoot -d /home/spiderfoot -s /sbin/nologin \-c "SpiderFoot User" spiderfoot-ENV SPIDERFOOT_LOGS /home/spiderfoot/log+# Place database and logs outside installation directoryENV SPIDERFOOT_DATA /var/lib/spiderfoot-RUN mkdir -p "$SPIDERFOOT_LOGS" "$SPIDERFOOT_DATA" \- && chown spiderfoot:spiderfoot "$SPIDERFOOT_LOGS" "$SPIDERFOOT_DATA"+ENV SPIDERFOOT_LOGS /var/lib/spiderfoot/log+ENV SPIDERFOOT_CACHE /var/lib/spiderfoot/cache++RUN mkdir -p $SPIDERFOOT_DATA || true \+ && mkdir -p $SPIDERFOOT_LOGS || true \+ && mkdir -p $SPIDERFOOT_CACHE || true \+ && chown spiderfoot:spiderfoot $SPIDERFOOT_DATA \+ && chown spiderfoot:spiderfoot $SPIDERFOOT_LOGS \+ && chown spiderfoot:spiderfoot $SPIDERFOOT_CACHEWORKDIR /home/spiderfootCOPY . .ENV VIRTUAL_ENV=/opt/venvRUN mkdir -p "$VIRTUAL_ENV" || true-RUN chown spiderfoot:spiderfoot "$VIRTUAL_ENV"+ENV PATH="$VIRTUAL_ENV/bin:$PATH"+RUN python -m venv "$VIRTUAL_ENV"++ARG REQUIREMENTS=requirements.txt+COPY "$REQUIREMENTS" requirements.txtRUN chown -R spiderfoot:spiderfoot /tools-WORKDIR /tools+RUN chown -R spiderfoot:spiderfoot "$VIRTUAL_ENV"+RUN chown -R spiderfoot:spiderfoot "/home/spiderfoot"USER spiderfoot-ARG REQUIREMENTS=requirements.txt-COPY "$REQUIREMENTS" requirements.txt-ENV PATH="$VIRTUAL_ENV/bin:$PATH"-RUN python -m venv "$VIRTUAL_ENV"RUN pip install -U pipRUN pip install -r "$REQUIREMENTS"# Install Python toolsRUN pip install dnstwist# CMSeeK+WORKDIR /toolsRUN git clone https://github.com/Tuhinshubhra/CMSeeK && cd CMSeeK \- && pip install -r requirements.txt && mkdir Results && cd /tools+ && pip install -r requirements.txt && mkdir ResultsWORKDIR /home/spiderfootEXPOSE 5001-# Configure tool paths-ENTRYPOINT python -c 'from spiderfoot import SpiderFootDb; \-db = SpiderFootDb({"__database": "spiderfoot.db"}, init=True); \+# Run the application+CMD python -c 'from spiderfoot import SpiderFootDb; \+db = SpiderFootDb({"__database": "/var/lib/spiderfoot/spiderfoot.db"}); \db.configSet({ \"sfp_tool_cmseek:cmseekpath": "/tools/CMSeeK/cmseek.py", \"sfp_tool_whatweb:whatweb_path": "/tools/WhatWeb/whatweb" \-})' && /bin/bash--# Run the application.-CMD ["./sf.py", "-l", "0.0.0.0:5001"]\ No newline at end of file+})' || true && ./sf.py -l 0.0.0.0:5001diff --git a/docker-compose.yml b/docker-compose.ymlindex aabe1a2a..4069c2d0 100644--- a/docker-compose.yml+++ b/docker-compose.yml@@ -8,15 +8,25 @@ version: "3"## Full image (all CLI tools installed):# $ docker-compose -f docker-compose.yml -f docker-compose-full.yml up+#+# Spiderfoot data resides in a Docker volume+#+# $ ls -lh /var/lib/docker/volumes/spiderfoot_spiderfoot-data/_data+# total 104K+# drwxr-xr-x 2 user user 4.0K Sep 22 09:51 cache+# -rw-r--r-- 1 user user 100K Sep 22 15:19 spiderfoot.dbservices:spiderfoot:build:- context: ./+ context: ./volumes:- - ./spiderfoot.db:/home/spiderfoot/spiderfoot.db+ - spiderfoot-data:/var/lib/spiderfootimage: spiderfootcontainer_name: spiderfootports:- "5001:5001"restart: unless-stopped++volumes:+ spiderfoot-data:diff --git a/spiderfoot/db.py b/spiderfoot/db.pyindex f132eff0..9ddc5e39 100644--- a/spiderfoot/db.py+++ b/spiderfoot/db.py@@ -10,6 +10,7 @@# Licence: GPL# -------------------------------------------------------------------------------+from pathlib import Pathimport reimport sqlite3import threading@@ -276,6 +277,9 @@ class SpiderFootDb:database_path = opts['__database']+ # create database directory+ Path(database_path).parent.mkdir(exist_ok=True, parents=True)+# connect() will create the database file if it doesn't exist, but# at least we can use this opportunity to ensure we have permissions to# read and write to such a file.