Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(self-host): rework docker-compose.yaml

Gergő Móriczd53371831261
diff --git a/docker-compose.yaml b/docker-compose.yaml
index 0fc22fcaa..32179b948 100644
--- a/docker-compose.yaml
+++ b/docker-compose.yaml
@@ -1,7 +1,11 @@
name: firecrawl
x-common-service: &common-service
- build: apps/api
+ # NOTE: If you want to build the service locally,
+ # uncomment the build: statement and comment out the image: statement
+ image: ghcr.io/mendableai/firecrawl
+ # build: apps/api
+
ulimits:
nofile:
soft: 65535
@@ -11,6 +15,31 @@ x-common-service: &common-service
extra_hosts:
- "host.docker.internal:host-gateway"
+x-common-env: &common-env
+ REDIS_URL: ${REDIS_URL:-redis://redis:6379}
+ REDIS_RATE_LIMIT_URL: ${REDIS_URL:-redis://redis:6379}
+ PLAYWRIGHT_MICROSERVICE_URL: ${PLAYWRIGHT_MICROSERVICE_URL:-http://playwright-service:3000/scrape}
+ USE_DB_AUTHENTICATION: ${USE_DB_AUTHENTICATION}
+ OPENAI_API_KEY: ${OPENAI_API_KEY}
+ OPENAI_BASE_URL: ${OPENAI_BASE_URL}
+ MODEL_NAME: ${MODEL_NAME}
+ SLACK_WEBHOOK_URL: ${SLACK_WEBHOOK_URL}
+ BULL_AUTH_KEY: ${BULL_AUTH_KEY}
+ TEST_API_KEY: ${TEST_API_KEY}
+ POSTHOG_API_KEY: ${POSTHOG_API_KEY}
+ POSTHOG_HOST: ${POSTHOG_HOST}
+ SUPABASE_ANON_TOKEN: ${SUPABASE_ANON_TOKEN}
+ SUPABASE_URL: ${SUPABASE_URL}
+ SUPABASE_SERVICE_TOKEN: ${SUPABASE_SERVICE_TOKEN}
+ SCRAPING_BEE_API_KEY: ${SCRAPING_BEE_API_KEY}
+ SELF_HOSTED_WEBHOOK_URL: ${SELF_HOSTED_WEBHOOK_URL}
+ SERPER_API_KEY: ${SERPER_API_KEY}
+ SEARCHAPI_API_KEY: ${SEARCHAPI_API_KEY}
+ LOGGING_LEVEL: ${LOGGING_LEVEL}
+ PROXY_SERVER: ${PROXY_SERVER}
+ PROXY_USERNAME: ${PROXY_USERNAME}
+ PROXY_PASSWORD: ${PROXY_PASSWORD}
+
services:
playwright-service:
build: apps/playwright-service-ts
@@ -26,71 +55,21 @@ services:
api:
<<: *common-service
environment:
- REDIS_URL: ${REDIS_URL:-redis://redis:6379}
- REDIS_RATE_LIMIT_URL: ${REDIS_URL:-redis://redis:6379}
- PLAYWRIGHT_MICROSERVICE_URL: ${PLAYWRIGHT_MICROSERVICE_URL:-http://playwright-service:3000/scrape}
- USE_DB_AUTHENTICATION: ${USE_DB_AUTHENTICATION}
- PORT: ${PORT:-3002}
- NUM_WORKERS_PER_QUEUE: ${NUM_WORKERS_PER_QUEUE}
- OPENAI_API_KEY: ${OPENAI_API_KEY}
- OPENAI_BASE_URL: ${OPENAI_BASE_URL}
- MODEL_NAME: ${MODEL_NAME:-gpt-4o}
- SLACK_WEBHOOK_URL: ${SLACK_WEBHOOK_URL}
- LLAMAPARSE_API_KEY: ${LLAMAPARSE_API_KEY}
- LOGTAIL_KEY: ${LOGTAIL_KEY}
- BULL_AUTH_KEY: ${BULL_AUTH_KEY}
- TEST_API_KEY: ${TEST_API_KEY}
- POSTHOG_API_KEY: ${POSTHOG_API_KEY}
- POSTHOG_HOST: ${POSTHOG_HOST}
- SUPABASE_ANON_TOKEN: ${SUPABASE_ANON_TOKEN}
- SUPABASE_URL: ${SUPABASE_URL}
- SUPABASE_SERVICE_TOKEN: ${SUPABASE_SERVICE_TOKEN}
- SCRAPING_BEE_API_KEY: ${SCRAPING_BEE_API_KEY}
+ <<: *common-env
HOST: ${HOST:-0.0.0.0}
- SELF_HOSTED_WEBHOOK_URL: ${SELF_HOSTED_WEBHOOK_URL}
- SERPER_API_KEY: ${SERPER_API_KEY}
- SEARCHAPI_API_KEY: ${SEARCHAPI_API_KEY}
- LOGGING_LEVEL: ${LOGGING_LEVEL}
- PROXY_SERVER: ${PROXY_SERVER}
- PROXY_USERNAME: ${PROXY_USERNAME}
- PROXY_PASSWORD: ${PROXY_PASSWORD}
+ PORT: 3002
FLY_PROCESS_GROUP: app
depends_on:
- redis
- playwright-service
ports:
- - "3002:3002"
+ - "${PORT:-3002}:3002"
command: [ "pnpm", "run", "start:production" ]
worker:
<<: *common-service
environment:
- REDIS_URL: ${REDIS_URL:-redis://redis:6379}
- REDIS_RATE_LIMIT_URL: ${REDIS_URL:-redis://redis:6379}
- PLAYWRIGHT_MICROSERVICE_URL: ${PLAYWRIGHT_MICROSERVICE_URL:-http://playwright-service:3000/scrape}
- USE_DB_AUTHENTICATION: ${USE_DB_AUTHENTICATION}
- PORT: ${PORT:-3002}
- NUM_WORKERS_PER_QUEUE: ${NUM_WORKERS_PER_QUEUE}
- OPENAI_API_KEY: ${OPENAI_API_KEY}
- OPENAI_BASE_URL: ${OPENAI_BASE_URL}
- MODEL_NAME: ${MODEL_NAME:-gpt-4o}
- SLACK_WEBHOOK_URL: ${SLACK_WEBHOOK_URL}
- LLAMAPARSE_API_KEY: ${LLAMAPARSE_API_KEY}
- LOGTAIL_KEY: ${LOGTAIL_KEY}
- BULL_AUTH_KEY: ${BULL_AUTH_KEY}
- TEST_API_KEY: ${TEST_API_KEY}
- POSTHOG_API_KEY: ${POSTHOG_API_KEY}
- POSTHOG_HOST: ${POSTHOG_HOST}
- SUPABASE_ANON_TOKEN: ${SUPABASE_ANON_TOKEN}
- SUPABASE_URL: ${SUPABASE_URL}
- SUPABASE_SERVICE_TOKEN: ${SUPABASE_SERVICE_TOKEN}
- SCRAPING_BEE_API_KEY: ${SCRAPING_BEE_API_KEY}
- HOST: ${HOST:-0.0.0.0}
- SELF_HOSTED_WEBHOOK_URL: ${SELF_HOSTED_WEBHOOK_URL}
- LOGGING_LEVEL: ${LOGGING_LEVEL}
- PROXY_SERVER: ${PROXY_SERVER}
- PROXY_USERNAME: ${PROXY_USERNAME}
- PROXY_PASSWORD: ${PROXY_PASSWORD}
+ <<: *common-env
FLY_PROCESS_GROUP: worker
depends_on:
- redis