Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

Merge branch 'main' into chadha93/self-host-concurrency

Gaurav Chadha11c3f6506cb3
diff --cc docker-compose.yaml
index 109c989e9,184e1d60e..30731f341
--- a/docker-compose.yaml
+++ b/docker-compose.yaml
@@@ -19,12 -25,12 +25,16 @@@ x-common-env: &common-en
REDIS_URL: ${REDIS_URL:-redis://redis:6379}
REDIS_RATE_LIMIT_URL: ${REDIS_URL:-redis://redis:6379}
PLAYWRIGHT_MICROSERVICE_URL: ${PLAYWRIGHT_MICROSERVICE_URL:-http://playwright-service:3000/scrape}
- NUQ_DATABASE_URL: postgres://postgres:postgres@nuq-postgres:5432/postgres
+ POSTGRES_USER: ${POSTGRES_USER:-postgres}
+ POSTGRES_PASSWORD: "${POSTGRES_PASSWORD:-postgres}"
+ POSTGRES_DB: ${POSTGRES_DB:-postgres}
+ POSTGRES_HOST: ${POSTGRES_HOST:-nuq-postgres}
+ POSTGRES_PORT: ${POSTGRES_PORT:-5432}
USE_DB_AUTHENTICATION: ${USE_DB_AUTHENTICATION}
+ NUM_WORKERS_PER_QUEUE: ${NUM_WORKERS_PER_QUEUE:-8}
+ CRAWL_CONCURRENT_REQUESTS: ${CRAWL_CONCURRENT_REQUESTS:-10}
+ MAX_CONCURRENT_JOBS: ${MAX_CONCURRENT_JOBS:-5}
+ BROWSER_POOL_SIZE: ${BROWSER_POOL_SIZE:-5}
OPENAI_API_KEY: ${OPENAI_API_KEY}
OPENAI_BASE_URL: ${OPENAI_BASE_URL}
MODEL_NAME: ${MODEL_NAME}
@@@ -61,14 -65,16 +69,22 @@@ services
PROXY_USERNAME: ${PROXY_USERNAME}
PROXY_PASSWORD: ${PROXY_PASSWORD}
BLOCK_MEDIA: ${BLOCK_MEDIA}
+ # Configure maximum concurrent pages for Playwright browser instances
+ MAX_CONCURRENT_PAGES: ${CRAWL_CONCURRENT_REQUESTS:-10}
networks:
- backend
+ # Resource limits for Docker Compose (not Swarm)
+ cpus: 2.0
+ mem_limit: 4G
+ memswap_limit: 4G
+ logging:
+ driver: "json-file"
+ options:
+ max-size: "10m"
+ max-file: "3"
+ compress: "true"
+ tmpfs:
+ - /tmp/.cache:noexec,nosuid,size=1g
api:
<<: *common-service