Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
address-bot-feedback
Gaurav Chadhaacd49389b137
diff --git a/apps/api/.env.example b/apps/api/.env.exampleindex a1f87e87a..3ba32de3c 100644--- a/apps/api/.env.example+++ b/apps/api/.env.example@@ -16,7 +16,7 @@ SEARCHAPI_API_KEY=# SearchApi engine, defaults to google. Available options: google, bing, baidu, google_news, etc. Head to https://searchapi.com/ to explore more enginesSEARCHAPI_ENGINE=-## === Concurrency & Performance Settings ===+## === Concurrency & Performance Settings ===\n# Number of worker processes per queue - controls parallel job processing\n# Recommended: 4-8 for most setups, adjust based on CPU cores\nNUM_WORKERS_PER_QUEUE=8# Number of concurrent crawl requests/pages - increase for higher throughput# Recommended: 8-16 for most setups, 32+ for high-performance serversCRAWL_CONCURRENT_REQUESTS=10@@ -43,7 +43,7 @@ RATE_LIMIT_TEST_API_KEY_SCRAPE=RATE_LIMIT_TEST_API_KEY_CRAWL=# set if you'd like to use scraping Be to handle JS blockingSCRAPING_BEE_API_KEY=-# add for LLM dependednt features (image alt generation, etc.)+# add for LLM dependent features (image alt generation, etc.)OPENAI_API_KEY=BULL_AUTH_KEY=@# set if you have a llamaparse key you'd like to use to parse pdfsdiff --git a/docker-compose.yaml b/docker-compose.yamlindex 4e275b231..109c989e9 100644--- a/docker-compose.yaml+++ b/docker-compose.yaml@@ -65,14 +65,10 @@ services:MAX_CONCURRENT_PAGES: ${CRAWL_CONCURRENT_REQUESTS:-10}networks:- backend- deploy:- resources:- limits:- cpus: '2.0'- memory: 4G- reservations:- cpus: '1.0'- memory: 2G+ # Resource limits for Docker Compose (not Swarm)+ cpus: 2.0+ mem_limit: 4G+ memswap_limit: 4Gapi:<<: *common-service@@ -88,14 +84,11 @@ services:ports:- "${PORT:-3002}:${INTERNAL_PORT:-3002}"command: node dist/src/harness.js --start-docker- deploy:- resources:- limits:- cpus: '4.0' # Increase if you have more CPU cores- memory: 8G # Increase if you have more RAM- reservations:- cpus: '2.0' # Minimum guaranteed CPU- memory: 4G # Minimum guaranteed RAM+ # Resource limits for Docker Compose (not Swarm)+ # Increase if you have more CPU cores/RAM available+ cpus: 4.0+ mem_limit: 8G+ memswap_limit: 8Gredis:# NOTE: If you want to use Valkey (open source) instead of Redis (source available),