A modern platform for visual, flexible, and extensible graph-based investigations. For cybersecurity analysts and investigators.
feat: upgrade deploy to use docker only
dextmorgnd0417e0056c5
diff --git a/.dockerignore b/.dockerignorenew file mode 100644index 00000000..b8594be8--- /dev/null+++ b/.dockerignore@@ -0,0 +1,73 @@+# Python+__pycache__/+*.py[cod]+*$py.class+*.so+.Python+*.egg-info/+dist/+build/+*.egg+.eggs/++# Virtual environments+.venv/+venv/+ENV/+env/++# IDE+.vscode/+.idea/+*.swp+*.swo+*~+.DS_Store++# Git+.git/+.gitignore+.gitattributes++# Documentation+*.md+!README.md+docs/++# Testing+.pytest_cache/+.coverage+htmlcov/+.tox/+.mypy_cache/+.hypothesis/++# Node (for frontend)+flowsint-app/node_modules/+flowsint-app/dist/+flowsint-app/.next/+flowsint-app/out/+flowsint-app/build/++# Environment files+.env.local+.env.*.local++# Logs+*.log+logs/++# Docker+docker-compose*.yml+Dockerfile.dev+.dockerignore++# CI/CD+.github/+.gitlab-ci.yml++# Misc+*.bak+*.tmp+temp/+tmp/diff --git a/.env.example b/.env.exampleindex aa6db3d4..cc6fc2b5 100644--- a/.env.example+++ b/.env.example@@ -5,6 +5,7 @@ NEO4J_USERNAME=neo4jNEO4J_PASSWORD=passwordVITE_API_URL=http://127.0.0.1:5001DATABASE_URL=postgresql://flowsint:flowsint@localhost:5433/flowsint+REDIS_URL=redis://redis:6379/0HIBP_API_KEY=ETHERSCAN_API_KEY=WHOXY_API_KEY=diff --git a/Makefile b/Makefileindex 55b023a3..dbab9d9d 100644--- a/Makefile+++ b/Makefile@@ -1,19 +1,34 @@PROJECT_ROOT := $(shell pwd)-.PHONY: install run stop infra api frontend celery clean dev check-env+.PHONY: install run stop stop-dev stop-prod infra api frontend celery clean dev prod check-env build-dev build-prod open-browserENV_DIRS := . flowsint-api flowsint-core flowsint-app+open-browser:+ @echo "⏳ Waiting for frontend to be ready..."+ @bash -c 'until curl -s http://localhost:5173 > /dev/null 2>&1; do sleep 1; done'+ @echo "🌐 Opening browser..."+ @open http://localhost:5173 2>/dev/null || xdg-open http://localhost:5173 2>/dev/null || echo "✅ Flowsint ready at http://localhost:5173"dev:+ @echo "🚀 Starting Flowsint in DEVELOPMENT mode..."$(MAKE) check-env- $(MAKE) install- $(MAKE) run+ docker compose -f docker-compose.dev.yml up --build -d+ $(MAKE) open-browser+ docker compose -f docker-compose.dev.yml logs -fprod:- # temporary+ @echo "🚀 Starting Flowsint in PRODUCTION mode..."$(MAKE) check-env- $(MAKE) install- $(MAKE) run+ docker compose -f docker-compose.prod.yml up --build -d+ $(MAKE) open-browser++build-dev:+ @echo "🔨 Building development images..."+ docker compose -f docker-compose.dev.yml build++build-prod:+ @echo "🔨 Building production images..."+ docker compose -f docker-compose.prod.yml buildcheck-env:@echo "🔎 Checking .env files..."@@ -68,18 +83,30 @@ run:@echo "⏳ Waiting for frontend to be ready..."@bash -c 'until curl -s http://localhost:5173 > /dev/null 2>&1; do sleep 1; done'@echo "🌐 Opening browser..."- @open http://localhost:5173 2>/dev/null || xdg-open http://localhost:5173 2>/dev/null || echo "✅ All services ready! Frontend at http://localhost:5173"+ @open http://localhost:5173 2>/dev/null || xdg-open http://localhost:5173 2>/dev/null || echo "✅ All services ready! Flowsint at http://localhost:5173"$(MAKE) -j2 api celerystop:@echo "🛑 Stopping all services..."+ -docker compose -f docker-compose.dev.yml down+ -docker compose -f docker-compose.prod.yml down-docker compose down-# --- Nettoyage complet ---+stop-dev:+ @echo "🛑 Stopping development services..."+ docker compose -f docker-compose.dev.yml down++stop-prod:+ @echo "🛑 Stopping production services..."+ docker compose -f docker-compose.prod.yml down+clean:- @echo "Removing containers, volumes and venvs..."- docker compose down -v --remove-orphans+ @echo "🧹 Removing containers, volumes and venvs..."+ -docker compose -f docker-compose.dev.yml down -v --remove-orphans+ -docker compose -f docker-compose.prod.yml down -v --remove-orphans+ -docker compose down -v --remove-orphansrm -rf $(PROJECT_ROOT)/flowsint-app/node_modulesrm -rf $(PROJECT_ROOT)/flowsint-core/.venvrm -rf $(PROJECT_ROOT)/flowsint-transforms/.venvrm -rf $(PROJECT_ROOT)/flowsint-api/.venv+ @echo "✅ Cleanup complete!"diff --git a/docker-compose.dev.yml b/docker-compose.dev.ymlnew file mode 100644index 00000000..f4481cdc--- /dev/null+++ b/docker-compose.dev.yml@@ -0,0 +1,159 @@+name: flowsint-dev++services:+ # PostgreSQL database+ postgres:+ image: postgres:15+ container_name: flowsint-postgres-dev+ restart: always+ environment:+ POSTGRES_USER: flowsint+ POSTGRES_PASSWORD: flowsint+ POSTGRES_DB: flowsint+ ports:+ - "5433:5432"+ volumes:+ - pg_data_dev:/var/lib/postgresql/data+ networks:+ - flowsint_network+ healthcheck:+ test: ["CMD-SHELL", "pg_isready -U flowsint"]+ interval: 10s+ timeout: 5s+ retries: 5++ # Redis for Celery & cache+ redis:+ image: redis:alpine+ container_name: flowsint-redis-dev+ ports:+ - "6379:6379"+ networks:+ - flowsint_network+ healthcheck:+ test: ["CMD", "redis-cli", "ping"]+ interval: 10s+ timeout: 5s+ retries: 5++ # Neo4j graph database+ neo4j:+ image: neo4j:5+ container_name: flowsint-neo4j-dev+ ports:+ - "7474:7474" # Web UI+ - "7687:7687" # Bolt+ environment:+ - NEO4J_AUTH=${NEO4J_USERNAME}/${NEO4J_PASSWORD}+ - NEO4J_PLUGINS=["apoc"]+ - NEO4J_apoc_export_file_enabled=true+ - NEO4J_apoc_import_file_enabled=true+ - NEO4J_apoc_import_file_use__neo4j__config=true+ volumes:+ - neo4j_data_dev:/data+ - neo4j_logs_dev:/logs+ - neo4j_import_dev:/var/lib/neo4j/import+ - neo4j_plugins_dev:/plugins+ restart: unless-stopped+ networks:+ - flowsint_network+ healthcheck:+ test: ["CMD-SHELL", "wget --no-verbose --tries=1 --spider http://localhost:7474 || exit 1"]+ interval: 10s+ timeout: 5s+ retries: 5++ # FastAPI Backend+ api:+ build:+ context: .+ dockerfile: flowsint-api/Dockerfile+ target: dev+ container_name: flowsint-api-dev+ restart: unless-stopped+ ports:+ - "5001:5001"+ volumes:+ - ./flowsint-api:/app/flowsint-api+ - ./flowsint-core:/app/flowsint-core+ - ./flowsint-types:/app/flowsint-types+ - ./flowsint-transforms:/app/flowsint-transforms+ environment:+ - DATABASE_URL=postgresql://flowsint:flowsint@postgres:5432/flowsint+ - NEO4J_URI_BOLT=bolt://neo4j:7687+ - NEO4J_USERNAME=${NEO4J_USERNAME}+ - NEO4J_PASSWORD=${NEO4J_PASSWORD}+ - AUTH_SECRET=${AUTH_SECRET}+ - HIBP_API_KEY=${HIBP_API_KEY}+ - ETHERSCAN_API_KEY=${ETHERSCAN_API_KEY}+ - WHOXY_API_KEY=${WHOXY_API_KEY}+ - REDIS_URL=redis://redis:6379/0+ depends_on:+ postgres:+ condition: service_healthy+ redis:+ condition: service_healthy+ neo4j:+ condition: service_healthy+ networks:+ - flowsint_network++ # Celery Worker+ celery:+ build:+ context: .+ dockerfile: flowsint-api/Dockerfile+ target: dev+ container_name: flowsint-celery-dev+ restart: unless-stopped+ command: celery -A flowsint_core.core.celery worker --loglevel=info --pool=solo+ volumes:+ - ./flowsint-api:/app/flowsint-api+ - ./flowsint-core:/app/flowsint-core+ - ./flowsint-types:/app/flowsint-types+ - ./flowsint-transforms:/app/flowsint-transforms+ environment:+ - DATABASE_URL=postgresql://flowsint:flowsint@postgres:5432/flowsint+ - NEO4J_URI_BOLT=bolt://neo4j:7687+ - NEO4J_USERNAME=${NEO4J_USERNAME}+ - NEO4J_PASSWORD=${NEO4J_PASSWORD}+ - REDIS_URL=redis://redis:6379/0+ depends_on:+ postgres:+ condition: service_healthy+ redis:+ condition: service_healthy+ neo4j:+ condition: service_healthy+ networks:+ - flowsint_network++ # Frontend+ app:+ build:+ context: ./flowsint-app+ dockerfile: Dockerfile.dev+ container_name: flowsint-app-dev+ ports:+ - "5173:5173"+ volumes:+ - ./flowsint-app:/app+ - /app/node_modules+ environment:+ - VITE_API_URL=http://localhost:5001+ networks:+ - flowsint_network+ stdin_open: true+ tty: true++networks:+ flowsint_network:+ name: flowsint_network_dev+ driver: bridge++volumes:+ pg_data_dev:+ neo4j_data_dev:+ neo4j_logs_dev:+ neo4j_import_dev:+ neo4j_plugins_dev:diff --git a/docker-compose.prod.yml b/docker-compose.prod.ymlnew file mode 100644index 00000000..6d497d63--- /dev/null+++ b/docker-compose.prod.yml@@ -0,0 +1,148 @@+name: flowsint-prod++services:+ # PostgreSQL database+ postgres:+ image: postgres:15+ container_name: flowsint-postgres-prod+ restart: always+ environment:+ POSTGRES_USER: ${POSTGRES_USER:-flowsint}+ POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-flowsint}+ POSTGRES_DB: ${POSTGRES_DB:-flowsint}+ ports:+ - "5433:5432"+ volumes:+ - pg_data_prod:/var/lib/postgresql/data+ networks:+ - flowsint_network+ healthcheck:+ test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER:-flowsint}"]+ interval: 10s+ timeout: 5s+ retries: 5++ # Redis for Celery & cache+ redis:+ image: redis:alpine+ container_name: flowsint-redis-prod+ restart: always+ ports:+ - "6379:6379"+ networks:+ - flowsint_network+ healthcheck:+ test: ["CMD", "redis-cli", "ping"]+ interval: 10s+ timeout: 5s+ retries: 5++ # Neo4j graph database+ neo4j:+ image: neo4j:5+ container_name: flowsint-neo4j-prod+ restart: always+ ports:+ - "7474:7474" # Web UI+ - "7687:7687" # Bolt+ environment:+ - NEO4J_AUTH=${NEO4J_USERNAME}/${NEO4J_PASSWORD}+ - NEO4J_PLUGINS=["apoc"]+ - NEO4J_apoc_export_file_enabled=true+ - NEO4J_apoc_import_file_enabled=true+ - NEO4J_apoc_import_file_use__neo4j__config=true+ volumes:+ - neo4j_data_prod:/data+ - neo4j_logs_prod:/logs+ - neo4j_import_prod:/var/lib/neo4j/import+ - neo4j_plugins_prod:/plugins+ networks:+ - flowsint_network+ healthcheck:+ test: ["CMD-SHELL", "wget --no-verbose --tries=1 --spider http://localhost:7474 || exit 1"]+ interval: 10s+ timeout: 5s+ retries: 5++ # FastAPI Backend (Production)+ api:+ build:+ context: .+ dockerfile: flowsint-api/Dockerfile+ target: prod+ container_name: flowsint-api-prod+ restart: always+ ports:+ - "5001:5001"+ environment:+ - DATABASE_URL=postgresql://${POSTGRES_USER:-flowsint}:${POSTGRES_PASSWORD:-flowsint}@postgres:5432/${POSTGRES_DB:-flowsint}+ - NEO4J_URI_BOLT=bolt://neo4j:7687+ - NEO4J_USERNAME=${NEO4J_USERNAME}+ - NEO4J_PASSWORD=${NEO4J_PASSWORD}+ - AUTH_SECRET=${AUTH_SECRET}+ - HIBP_API_KEY=${HIBP_API_KEY}+ - ETHERSCAN_API_KEY=${ETHERSCAN_API_KEY}+ - WHOXY_API_KEY=${WHOXY_API_KEY}+ - REDIS_URL=redis://redis:6379/0+ depends_on:+ postgres:+ condition: service_healthy+ redis:+ condition: service_healthy+ neo4j:+ condition: service_healthy+ networks:+ - flowsint_network++ # Celery Worker (Production)+ celery:+ build:+ context: .+ dockerfile: flowsint-api/Dockerfile+ target: prod+ container_name: flowsint-celery-prod+ restart: always+ command: celery -A flowsint_core.core.celery worker --loglevel=info --pool=solo+ environment:+ - DATABASE_URL=postgresql://${POSTGRES_USER:-flowsint}:${POSTGRES_PASSWORD:-flowsint}@postgres:5432/${POSTGRES_DB:-flowsint}+ - NEO4J_URI_BOLT=bolt://neo4j:7687+ - NEO4J_USERNAME=${NEO4J_USERNAME}+ - NEO4J_PASSWORD=${NEO4J_PASSWORD}+ - REDIS_URL=redis://redis:6379/0+ depends_on:+ postgres:+ condition: service_healthy+ redis:+ condition: service_healthy+ neo4j:+ condition: service_healthy+ networks:+ - flowsint_network++ # Frontend (Production)+ app:+ build:+ context: ./flowsint-app+ dockerfile: Dockerfile+ args:+ - VITE_API_URL=${VITE_API_URL:-http://localhost:5001}+ container_name: flowsint-app-prod+ restart: always+ ports:+ - "5173:5173"+ networks:+ - flowsint_network+ depends_on:+ - api++networks:+ flowsint_network:+ name: flowsint_network_prod+ driver: bridge++volumes:+ pg_data_prod:+ neo4j_data_prod:+ neo4j_logs_prod:+ neo4j_import_prod:+ neo4j_plugins_prod:diff --git a/flowsint-api/Dockerfile b/flowsint-api/Dockerfileindex edfd437a..50b7db10 100644--- a/flowsint-api/Dockerfile+++ b/flowsint-api/Dockerfile@@ -1,16 +1,18 @@-# --- Production stage ----FROM python:3.12-slim AS prod+# --- Base stage with Python 3.12 ---+FROM python:3.12-slim AS base# Environment variablesENV PYTHONUNBUFFERED=1ENV POETRY_VIRTUALENVS_CREATE=false-ENV APP_ENV=prod# Install system dependenciesRUN apt-get update && apt-get install -y \build-essential \curl \git \+ libpq-dev \+ pkg-config \+ libcairo2-dev \&& rm -rf /var/lib/apt/lists/*# Install Poetry@@ -19,17 +21,50 @@ RUN curl -sSL https://install.python-poetry.org | python3 - \WORKDIR /app-# Copy only dependency files to leverage Docker cache-COPY pyproject.toml poetry.lock ./+# --- Development stage ---+FROM base AS dev-# Install production dependencies only (no dev)-RUN poetry install --no-dev --no-root+ENV APP_ENV=development++# Copy all dependency packages+COPY flowsint-core /app/flowsint-core+COPY flowsint-types /app/flowsint-types+COPY flowsint-transforms /app/flowsint-transforms++# Copy API files+COPY flowsint-api /app/flowsint-api++WORKDIR /app/flowsint-api++# Install dependencies with dev packages+RUN poetry install --no-root++# Expose FastAPI port+EXPOSE 5001-# Copy the source code-COPY . .+ENTRYPOINT ["/app/flowsint-api/entrypoint.sh"]+CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "5001", "--reload"]++# --- Production stage ---+FROM base AS prod++ENV APP_ENV=production++# Copy all dependency packages+COPY flowsint-core /app/flowsint-core+COPY flowsint-types /app/flowsint-types+COPY flowsint-transforms /app/flowsint-transforms++# Copy API files+COPY flowsint-api /app/flowsint-api++WORKDIR /app/flowsint-api++# Install production dependencies only (no dev)+RUN poetry install# Expose FastAPI portEXPOSE 5001-# Run FastAPI+ENTRYPOINT ["/app/flowsint-api/entrypoint.sh"]CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "5001"]diff --git a/flowsint-api/entrypoint.sh b/flowsint-api/entrypoint.shindex c36a0f65..d77c9ba8 100755--- a/flowsint-api/entrypoint.sh+++ b/flowsint-api/entrypoint.sh@@ -1,12 +1,8 @@#!/bin/sh-set -e # Arrêter le script en cas d'erreur+set -e-if [ "$1" = "app" ]; then- echo "Démarrage de FastAPI..."- exec uvicorn app.main:app --host 0.0.0.0 --port 5001-elif [ "$1" = "celery" ]; then- echo "Démarrage de Celery..."- exec celery -A app.core.celery worker --loglevel=info-else- exec "$@"-fi+echo "Running database migrations..."+alembic upgrade head++echo "Starting application..."+exec "$@"diff --git a/flowsint-app/Dockerfile b/flowsint-app/Dockerfileindex 956cd3f7..68d56ea0 100644--- a/flowsint-app/Dockerfile+++ b/flowsint-app/Dockerfile@@ -1,27 +1,13 @@-# Build stage-FROM node:20 AS builder+FROM node:24WORKDIR /app-# Copy package files and install dependenciesCOPY package*.json ./-RUN npm install --legacy-peer-deps+RUN yarn installCOPY . .-# Type check and build-RUN npm run typecheck-RUN npm run build+RUN yarn build+EXPOSE 5173-# Production stage-FROM nginx:alpine--# Copy built files to nginx-COPY --from=builder /app/dist /usr/share/nginx/html--# Copy nginx configuration-COPY nginx.conf /etc/nginx/conf.d/default.conf--EXPOSE 80--CMD ["nginx", "-g", "daemon off;"]+CMD ["yarn", "preview", "--", "--port", "5173"]diff --git a/flowsint-app/vite.config.ts b/flowsint-app/vite.config.tsindex 7b777ca6..15d0b1d1 100644--- a/flowsint-app/vite.config.ts+++ b/flowsint-app/vite.config.ts@@ -32,6 +32,7 @@ export default defineConfig({}},server: {+ open: true,proxy: {'/api': {target: 'http://localhost:5001',@@ -39,5 +40,10 @@ export default defineConfig({secure: false}}+ },+ preview: {+ open: false,+ host: '0.0.0.0',+ port: 5173}})diff --git a/flowsint-core/src/flowsint_core/core/config.py b/flowsint-core/src/flowsint_core/core/config.pyindex bec1c203..cc46c81b 100644--- a/flowsint-core/src/flowsint_core/core/config.py+++ b/flowsint-core/src/flowsint_core/core/config.py@@ -5,8 +5,8 @@ load_dotenv()class Settings:- CELERY_BROKER_URL = os.getenv("REDIS_URI", "redis://localhost:6379/0")- CELERY_RESULT_BACKEND = os.getenv("REDIS_URI", "redis://localhost:6379/0")+ CELERY_BROKER_URL = os.environ["REDIS_URL"]+ CELERY_RESULT_BACKEND = os.environ["REDIS_URL"]settings = Settings()diff --git a/flowsint-core/src/flowsint_core/core/events.py b/flowsint-core/src/flowsint_core/core/events.pyindex a2d9a88b..c2b633b6 100644--- a/flowsint-core/src/flowsint_core/core/events.py+++ b/flowsint-core/src/flowsint_core/core/events.py@@ -11,7 +11,7 @@ import osclass EventEmitter:def __init__(self):self.id = uuid.uuid4()- self.redis = redis.from_url(os.getenv("REDIS_URI", "redis://localhost:6379/0"))+ self.redis = redis.from_url(os.environ["REDIS_URL"])self.pubsubs: Dict[str, redis.client.PubSub] = {}async def subscribe(self, channel: str):diff --git a/flowsint-core/src/flowsint_core/core/graph_db.py b/flowsint-core/src/flowsint_core/core/graph_db.pyindex 84a256c2..7b28362e 100644--- a/flowsint-core/src/flowsint_core/core/graph_db.py+++ b/flowsint-core/src/flowsint_core/core/graph_db.py@@ -21,7 +21,6 @@ class Neo4jConnection:URI = os.getenv("NEO4J_URI_BOLT")-URI = "bolt://localhost:7687"USERNAME = os.getenv("NEO4J_USERNAME")PASSWORD = os.getenv("NEO4J_PASSWORD")diff --git a/flowsint-core/src/flowsint_core/tasks/event.py b/flowsint-core/src/flowsint_core/tasks/event.pyindex 2a2a6bb3..9f421eba 100644--- a/flowsint-core/src/flowsint_core/tasks/event.py+++ b/flowsint-core/src/flowsint_core/tasks/event.py@@ -17,9 +17,7 @@ def emit_event_task(log_id: str, sketch_id: str, log_type: EventLevel, content:event = Event(id=log_id, sketch_id=sketch_id, type=log_type, payload=content).model_dump_json()- redis_client = redis.from_url(- os.getenv("REDIS_URI", "redis://127.0.0.1:6379/0")- )+ redis_client = redis.from_url(os.environ["REDIS_URL"])redis_client.publish(sketch_id, event)except Exception as e:raisediff --git a/flowsint-core/src/flowsint_core/tasks/flow.py b/flowsint-core/src/flowsint_core/tasks/flow.pyindex 0b718f22..e55683c9 100644--- a/flowsint-core/src/flowsint_core/tasks/flow.py+++ b/flowsint-core/src/flowsint_core/tasks/flow.py@@ -18,7 +18,6 @@ from flowsint_core.utils import to_json_serializableload_dotenv()URI = os.getenv("NEO4J_URI_BOLT")-URI = "bolt://localhost:7687"USERNAME = os.getenv("NEO4J_USERNAME")PASSWORD = os.getenv("NEO4J_PASSWORD")diff --git a/flowsint-core/src/flowsint_core/tasks/transform.py b/flowsint-core/src/flowsint_core/tasks/transform.pyindex 4a714bb6..fe12b5a8 100644--- a/flowsint-core/src/flowsint_core/tasks/transform.py+++ b/flowsint-core/src/flowsint_core/tasks/transform.py@@ -18,7 +18,6 @@ from flowsint_core.utils import to_json_serializableload_dotenv()URI = os.getenv("NEO4J_URI_BOLT")-URI = "bolt://localhost:7687"USERNAME = os.getenv("NEO4J_USERNAME")PASSWORD = os.getenv("NEO4J_PASSWORD")