Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Merge branch 'main' into chadha93/self-host-concurrency
Gaurav Chadha11c3f6506cb3
diff --cc docker-compose.yamlindex 109c989e9,184e1d60e..30731f341--- a/docker-compose.yaml+++ b/docker-compose.yaml@@@ -19,12 -25,12 +25,16 @@@ x-common-env: &common-enREDIS_URL: ${REDIS_URL:-redis://redis:6379}REDIS_RATE_LIMIT_URL: ${REDIS_URL:-redis://redis:6379}PLAYWRIGHT_MICROSERVICE_URL: ${PLAYWRIGHT_MICROSERVICE_URL:-http://playwright-service:3000/scrape}- NUQ_DATABASE_URL: postgres://postgres:postgres@nuq-postgres:5432/postgres+ POSTGRES_USER: ${POSTGRES_USER:-postgres}+ POSTGRES_PASSWORD: "${POSTGRES_PASSWORD:-postgres}"+ POSTGRES_DB: ${POSTGRES_DB:-postgres}+ POSTGRES_HOST: ${POSTGRES_HOST:-nuq-postgres}+ POSTGRES_PORT: ${POSTGRES_PORT:-5432}USE_DB_AUTHENTICATION: ${USE_DB_AUTHENTICATION}+ NUM_WORKERS_PER_QUEUE: ${NUM_WORKERS_PER_QUEUE:-8}+ CRAWL_CONCURRENT_REQUESTS: ${CRAWL_CONCURRENT_REQUESTS:-10}+ MAX_CONCURRENT_JOBS: ${MAX_CONCURRENT_JOBS:-5}+ BROWSER_POOL_SIZE: ${BROWSER_POOL_SIZE:-5}OPENAI_API_KEY: ${OPENAI_API_KEY}OPENAI_BASE_URL: ${OPENAI_BASE_URL}MODEL_NAME: ${MODEL_NAME}@@@ -61,14 -65,16 +69,22 @@@ servicesPROXY_USERNAME: ${PROXY_USERNAME}PROXY_PASSWORD: ${PROXY_PASSWORD}BLOCK_MEDIA: ${BLOCK_MEDIA}+ # Configure maximum concurrent pages for Playwright browser instances+ MAX_CONCURRENT_PAGES: ${CRAWL_CONCURRENT_REQUESTS:-10}networks:- backend+ # Resource limits for Docker Compose (not Swarm)+ cpus: 2.0+ mem_limit: 4G+ memswap_limit: 4G+ logging:+ driver: "json-file"+ options:+ max-size: "10m"+ max-file: "3"+ compress: "true"+ tmpfs:+ - /tmp/.cache:noexec,nosuid,size=1gapi:<<: *common-service