Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(self-host): rework docker-compose.yaml
Gergő Móriczd53371831261
diff --git a/docker-compose.yaml b/docker-compose.yamlindex 0fc22fcaa..32179b948 100644--- a/docker-compose.yaml+++ b/docker-compose.yaml@@ -1,7 +1,11 @@name: firecrawlx-common-service: &common-service- build: apps/api+ # NOTE: If you want to build the service locally,+ # uncomment the build: statement and comment out the image: statement+ image: ghcr.io/mendableai/firecrawl+ # build: apps/api+ulimits:nofile:soft: 65535@@ -11,6 +15,31 @@ x-common-service: &common-serviceextra_hosts:- "host.docker.internal:host-gateway"+x-common-env: &common-env+ REDIS_URL: ${REDIS_URL:-redis://redis:6379}+ REDIS_RATE_LIMIT_URL: ${REDIS_URL:-redis://redis:6379}+ PLAYWRIGHT_MICROSERVICE_URL: ${PLAYWRIGHT_MICROSERVICE_URL:-http://playwright-service:3000/scrape}+ USE_DB_AUTHENTICATION: ${USE_DB_AUTHENTICATION}+ OPENAI_API_KEY: ${OPENAI_API_KEY}+ OPENAI_BASE_URL: ${OPENAI_BASE_URL}+ MODEL_NAME: ${MODEL_NAME}+ SLACK_WEBHOOK_URL: ${SLACK_WEBHOOK_URL}+ BULL_AUTH_KEY: ${BULL_AUTH_KEY}+ TEST_API_KEY: ${TEST_API_KEY}+ POSTHOG_API_KEY: ${POSTHOG_API_KEY}+ POSTHOG_HOST: ${POSTHOG_HOST}+ SUPABASE_ANON_TOKEN: ${SUPABASE_ANON_TOKEN}+ SUPABASE_URL: ${SUPABASE_URL}+ SUPABASE_SERVICE_TOKEN: ${SUPABASE_SERVICE_TOKEN}+ SCRAPING_BEE_API_KEY: ${SCRAPING_BEE_API_KEY}+ SELF_HOSTED_WEBHOOK_URL: ${SELF_HOSTED_WEBHOOK_URL}+ SERPER_API_KEY: ${SERPER_API_KEY}+ SEARCHAPI_API_KEY: ${SEARCHAPI_API_KEY}+ LOGGING_LEVEL: ${LOGGING_LEVEL}+ PROXY_SERVER: ${PROXY_SERVER}+ PROXY_USERNAME: ${PROXY_USERNAME}+ PROXY_PASSWORD: ${PROXY_PASSWORD}+services:playwright-service:build: apps/playwright-service-ts@@ -26,71 +55,21 @@ services:api:<<: *common-serviceenvironment:- REDIS_URL: ${REDIS_URL:-redis://redis:6379}- REDIS_RATE_LIMIT_URL: ${REDIS_URL:-redis://redis:6379}- PLAYWRIGHT_MICROSERVICE_URL: ${PLAYWRIGHT_MICROSERVICE_URL:-http://playwright-service:3000/scrape}- USE_DB_AUTHENTICATION: ${USE_DB_AUTHENTICATION}- PORT: ${PORT:-3002}- NUM_WORKERS_PER_QUEUE: ${NUM_WORKERS_PER_QUEUE}- OPENAI_API_KEY: ${OPENAI_API_KEY}- OPENAI_BASE_URL: ${OPENAI_BASE_URL}- MODEL_NAME: ${MODEL_NAME:-gpt-4o}- SLACK_WEBHOOK_URL: ${SLACK_WEBHOOK_URL}- LLAMAPARSE_API_KEY: ${LLAMAPARSE_API_KEY}- LOGTAIL_KEY: ${LOGTAIL_KEY}- BULL_AUTH_KEY: ${BULL_AUTH_KEY}- TEST_API_KEY: ${TEST_API_KEY}- POSTHOG_API_KEY: ${POSTHOG_API_KEY}- POSTHOG_HOST: ${POSTHOG_HOST}- SUPABASE_ANON_TOKEN: ${SUPABASE_ANON_TOKEN}- SUPABASE_URL: ${SUPABASE_URL}- SUPABASE_SERVICE_TOKEN: ${SUPABASE_SERVICE_TOKEN}- SCRAPING_BEE_API_KEY: ${SCRAPING_BEE_API_KEY}+ <<: *common-envHOST: ${HOST:-0.0.0.0}- SELF_HOSTED_WEBHOOK_URL: ${SELF_HOSTED_WEBHOOK_URL}- SERPER_API_KEY: ${SERPER_API_KEY}- SEARCHAPI_API_KEY: ${SEARCHAPI_API_KEY}- LOGGING_LEVEL: ${LOGGING_LEVEL}- PROXY_SERVER: ${PROXY_SERVER}- PROXY_USERNAME: ${PROXY_USERNAME}- PROXY_PASSWORD: ${PROXY_PASSWORD}+ PORT: 3002FLY_PROCESS_GROUP: appdepends_on:- redis- playwright-serviceports:- - "3002:3002"+ - "${PORT:-3002}:3002"command: [ "pnpm", "run", "start:production" ]worker:<<: *common-serviceenvironment:- REDIS_URL: ${REDIS_URL:-redis://redis:6379}- REDIS_RATE_LIMIT_URL: ${REDIS_URL:-redis://redis:6379}- PLAYWRIGHT_MICROSERVICE_URL: ${PLAYWRIGHT_MICROSERVICE_URL:-http://playwright-service:3000/scrape}- USE_DB_AUTHENTICATION: ${USE_DB_AUTHENTICATION}- PORT: ${PORT:-3002}- NUM_WORKERS_PER_QUEUE: ${NUM_WORKERS_PER_QUEUE}- OPENAI_API_KEY: ${OPENAI_API_KEY}- OPENAI_BASE_URL: ${OPENAI_BASE_URL}- MODEL_NAME: ${MODEL_NAME:-gpt-4o}- SLACK_WEBHOOK_URL: ${SLACK_WEBHOOK_URL}- LLAMAPARSE_API_KEY: ${LLAMAPARSE_API_KEY}- LOGTAIL_KEY: ${LOGTAIL_KEY}- BULL_AUTH_KEY: ${BULL_AUTH_KEY}- TEST_API_KEY: ${TEST_API_KEY}- POSTHOG_API_KEY: ${POSTHOG_API_KEY}- POSTHOG_HOST: ${POSTHOG_HOST}- SUPABASE_ANON_TOKEN: ${SUPABASE_ANON_TOKEN}- SUPABASE_URL: ${SUPABASE_URL}- SUPABASE_SERVICE_TOKEN: ${SUPABASE_SERVICE_TOKEN}- SCRAPING_BEE_API_KEY: ${SCRAPING_BEE_API_KEY}- HOST: ${HOST:-0.0.0.0}- SELF_HOSTED_WEBHOOK_URL: ${SELF_HOSTED_WEBHOOK_URL}- LOGGING_LEVEL: ${LOGGING_LEVEL}- PROXY_SERVER: ${PROXY_SERVER}- PROXY_USERNAME: ${PROXY_USERNAME}- PROXY_PASSWORD: ${PROXY_PASSWORD}+ <<: *common-envFLY_PROCESS_GROUP: workerdepends_on:- redis