Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
support-for-concurrency-self-host-crawl
Chadha9374c1a133765a
diff --git a/apps/api/.env.example b/apps/api/.env.exampleindex 6b8a900cc..38dcb6f01 100644--- a/apps/api/.env.example+++ b/apps/api/.env.example@@ -16,6 +16,19 @@ SEARCHAPI_API_KEY=# SearchApi engine, defaults to google. Available options: google, bing, baidu, google_news, etc. Head to https://searchapi.com/ to explore more enginesSEARCHAPI_ENGINE=+## === Concurrency & Performance Settings ===+# Number of worker processes per queue - increase for higher throughput+# Recommended: 8-16 for most setups, 32+ for high-performance servers+CRAWL_CONCURRENT_REQUESTS=10++# Maximum number of simultaneous crawl jobs+# Controls how many crawl operations can run at the same time+MAX_CONCURRENT_JOBS=5++# Size of browser instance pool for Playwright service+# Higher values allow more parallel browser operations+BROWSER_POOL_SIZE=5+# Supabase Setup (used to support DB authentication, advanced logging, etc.)SUPABASE_ANON_TOKEN=SUPABASE_URL=diff --git a/docker-compose.yaml b/docker-compose.yamlindex 5892fa0db..4e275b231 100644--- a/docker-compose.yaml+++ b/docker-compose.yaml@@ -21,6 +21,10 @@ x-common-env: &common-envPLAYWRIGHT_MICROSERVICE_URL: ${PLAYWRIGHT_MICROSERVICE_URL:-http://playwright-service:3000/scrape}NUQ_DATABASE_URL: postgres://postgres:postgres@nuq-postgres:5432/postgresUSE_DB_AUTHENTICATION: ${USE_DB_AUTHENTICATION}+ NUM_WORKERS_PER_QUEUE: ${NUM_WORKERS_PER_QUEUE:-8}+ CRAWL_CONCURRENT_REQUESTS: ${CRAWL_CONCURRENT_REQUESTS:-10}+ MAX_CONCURRENT_JOBS: ${MAX_CONCURRENT_JOBS:-5}+ BROWSER_POOL_SIZE: ${BROWSER_POOL_SIZE:-5}OPENAI_API_KEY: ${OPENAI_API_KEY}OPENAI_BASE_URL: ${OPENAI_BASE_URL}MODEL_NAME: ${MODEL_NAME}@@ -57,8 +61,18 @@ services:PROXY_USERNAME: ${PROXY_USERNAME}PROXY_PASSWORD: ${PROXY_PASSWORD}BLOCK_MEDIA: ${BLOCK_MEDIA}+ # Configure maximum concurrent pages for Playwright browser instances+ MAX_CONCURRENT_PAGES: ${CRAWL_CONCURRENT_REQUESTS:-10}networks:- backend+ deploy:+ resources:+ limits:+ cpus: '2.0'+ memory: 4G+ reservations:+ cpus: '1.0'+ memory: 2Gapi:<<: *common-service@@ -74,6 +88,14 @@ services:ports:- "${PORT:-3002}:${INTERNAL_PORT:-3002}"command: node dist/src/harness.js --start-docker+ deploy:+ resources:+ limits:+ cpus: '4.0' # Increase if you have more CPU cores+ memory: 8G # Increase if you have more RAM+ reservations:+ cpus: '2.0' # Minimum guaranteed CPU+ memory: 4G # Minimum guaranteed RAMredis:# NOTE: If you want to use Valkey (open source) instead of Redis (source available),