Skip to content

SpiderFoot automates OSINT for threat intelligence and mapping your attack surface.

Docker updates, fixes (#1386)

TheTechromancer45f87b4e4f85
diff --git a/Dockerfile b/Dockerfile
index 7d715b64..5e316dba 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -51,10 +51,11 @@ RUN pip3 install -r "$REQUIREMENTS"
FROM alpine:3.13.0
WORKDIR /home/spiderfoot
-ENV SPIDERFOOT_LOGS /home/spiderfoot/log
-# Place database and configs outside installation directory
+# Place database and logs outside installation directory
ENV SPIDERFOOT_DATA /var/lib/spiderfoot
+ENV SPIDERFOOT_LOGS /var/lib/spiderfoot/log
+ENV SPIDERFOOT_CACHE /var/lib/spiderfoot/cache
# Run everything as one command so that only one layer is created
RUN apk --update --no-cache add python3 musl openssl libxslt tinyxml libxml2 jpeg zlib openjpeg \
@@ -64,10 +65,12 @@ RUN apk --update --no-cache add python3 musl openssl libxslt tinyxml libxml2 jpe
&& rm -rf /var/cache/apk/* \
&& rm -rf /lib/apk/db \
&& rm -rf /root/.cache \
- && mkdir $SPIDERFOOT_DATA \
- && mkdir $SPIDERFOOT_LOGS \
+ && mkdir -p $SPIDERFOOT_DATA || true \
+ && mkdir -p $SPIDERFOOT_LOGS || true \
+ && mkdir -p $SPIDERFOOT_CACHE || true \
+ && chown spiderfoot:spiderfoot $SPIDERFOOT_DATA \
&& chown spiderfoot:spiderfoot $SPIDERFOOT_LOGS \
- && chown spiderfoot:spiderfoot $SPIDERFOOT_DATA
+ && chown spiderfoot:spiderfoot $SPIDERFOOT_CACHE
COPY . .
COPY --from=build /opt/venv /opt/venv
diff --git a/Dockerfile.full b/Dockerfile.full
index 2c934751..efa16b07 100644
--- a/Dockerfile.full
+++ b/Dockerfile.full
@@ -16,7 +16,7 @@ RUN apt-get -y update && apt-get -y install nmap
RUN mkdir /tools || true
WORKDIR /tools
RUN git clone --depth=1 https://github.com/blechschmidt/massdns.git \
- && cd massdns && make && make install && cd /tools && rm -r massdns
+ && cd massdns && make && make install && cd /tools && rm -r massdns
# Install Golang tools
RUN apt-get -y update && apt-get -y install golang
@@ -30,54 +30,60 @@ RUN GO111MODULE=on go get -v github.com/projectdiscovery/shuffledns/cmd/shuffled
RUN apt-get -y update && apt-get -y install ruby ruby-dev bundler
# WhatWeb
RUN git clone https://github.com/urbanadventurer/WhatWeb \
- && gem install rchardet mongo json && cd /tools/WhatWeb \
- && bundle install && cd /tools
+ && gem install rchardet mongo json && cd /tools/WhatWeb \
+ && bundle install && cd /tools
RUN groupadd spiderfoot \
&& useradd -m -g spiderfoot -d /home/spiderfoot -s /sbin/nologin \
-c "SpiderFoot User" spiderfoot
-ENV SPIDERFOOT_LOGS /home/spiderfoot/log
+# Place database and logs outside installation directory
ENV SPIDERFOOT_DATA /var/lib/spiderfoot
-RUN mkdir -p "$SPIDERFOOT_LOGS" "$SPIDERFOOT_DATA" \
- && chown spiderfoot:spiderfoot "$SPIDERFOOT_LOGS" "$SPIDERFOOT_DATA"
+ENV SPIDERFOOT_LOGS /var/lib/spiderfoot/log
+ENV SPIDERFOOT_CACHE /var/lib/spiderfoot/cache
+
+RUN mkdir -p $SPIDERFOOT_DATA || true \
+ && mkdir -p $SPIDERFOOT_LOGS || true \
+ && mkdir -p $SPIDERFOOT_CACHE || true \
+ && chown spiderfoot:spiderfoot $SPIDERFOOT_DATA \
+ && chown spiderfoot:spiderfoot $SPIDERFOOT_LOGS \
+ && chown spiderfoot:spiderfoot $SPIDERFOOT_CACHE
WORKDIR /home/spiderfoot
COPY . .
ENV VIRTUAL_ENV=/opt/venv
RUN mkdir -p "$VIRTUAL_ENV" || true
-RUN chown spiderfoot:spiderfoot "$VIRTUAL_ENV"
+ENV PATH="$VIRTUAL_ENV/bin:$PATH"
+RUN python -m venv "$VIRTUAL_ENV"
+
+ARG REQUIREMENTS=requirements.txt
+COPY "$REQUIREMENTS" requirements.txt
RUN chown -R spiderfoot:spiderfoot /tools
-WORKDIR /tools
+RUN chown -R spiderfoot:spiderfoot "$VIRTUAL_ENV"
+RUN chown -R spiderfoot:spiderfoot "/home/spiderfoot"
USER spiderfoot
-ARG REQUIREMENTS=requirements.txt
-COPY "$REQUIREMENTS" requirements.txt
-ENV PATH="$VIRTUAL_ENV/bin:$PATH"
-RUN python -m venv "$VIRTUAL_ENV"
RUN pip install -U pip
RUN pip install -r "$REQUIREMENTS"
# Install Python tools
RUN pip install dnstwist
# CMSeeK
+WORKDIR /tools
RUN git clone https://github.com/Tuhinshubhra/CMSeeK && cd CMSeeK \
- && pip install -r requirements.txt && mkdir Results && cd /tools
+ && pip install -r requirements.txt && mkdir Results
WORKDIR /home/spiderfoot
EXPOSE 5001
-# Configure tool paths
-ENTRYPOINT python -c 'from spiderfoot import SpiderFootDb; \
-db = SpiderFootDb({"__database": "spiderfoot.db"}, init=True); \
+# Run the application
+CMD python -c 'from spiderfoot import SpiderFootDb; \
+db = SpiderFootDb({"__database": "/var/lib/spiderfoot/spiderfoot.db"}); \
db.configSet({ \
"sfp_tool_cmseek:cmseekpath": "/tools/CMSeeK/cmseek.py", \
"sfp_tool_whatweb:whatweb_path": "/tools/WhatWeb/whatweb" \
-})' && /bin/bash
-
-# Run the application.
-CMD ["./sf.py", "-l", "0.0.0.0:5001"]
\ No newline at end of file
+})' || true && ./sf.py -l 0.0.0.0:5001
diff --git a/docker-compose.yml b/docker-compose.yml
index aabe1a2a..4069c2d0 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -8,15 +8,25 @@ version: "3"
#
# Full image (all CLI tools installed):
# $ docker-compose -f docker-compose.yml -f docker-compose-full.yml up
+#
+# Spiderfoot data resides in a Docker volume
+#
+# $ ls -lh /var/lib/docker/volumes/spiderfoot_spiderfoot-data/_data
+# total 104K
+# drwxr-xr-x 2 user user 4.0K Sep 22 09:51 cache
+# -rw-r--r-- 1 user user 100K Sep 22 15:19 spiderfoot.db
services:
spiderfoot:
build:
- context: ./
+ context: ./
volumes:
- - ./spiderfoot.db:/home/spiderfoot/spiderfoot.db
+ - spiderfoot-data:/var/lib/spiderfoot
image: spiderfoot
container_name: spiderfoot
ports:
- "5001:5001"
restart: unless-stopped
+
+volumes:
+ spiderfoot-data:
diff --git a/spiderfoot/db.py b/spiderfoot/db.py
index f132eff0..9ddc5e39 100644
--- a/spiderfoot/db.py
+++ b/spiderfoot/db.py
@@ -10,6 +10,7 @@
# Licence: GPL
# -------------------------------------------------------------------------------
+from pathlib import Path
import re
import sqlite3
import threading
@@ -276,6 +277,9 @@ class SpiderFootDb:
database_path = opts['__database']
+ # create database directory
+ Path(database_path).parent.mkdir(exist_ok=True, parents=True)
+
# connect() will create the database file if it doesn't exist, but
# at least we can use this opportunity to ensure we have permissions to
# read and write to such a file.