Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

address-bot-feedback

Gaurav Chadhaacd49389b137
diff --git a/apps/api/.env.example b/apps/api/.env.example
index a1f87e87a..3ba32de3c 100644
--- a/apps/api/.env.example
+++ b/apps/api/.env.example
@@ -16,7 +16,7 @@ SEARCHAPI_API_KEY=
# SearchApi engine, defaults to google. Available options: google, bing, baidu, google_news, etc. Head to https://searchapi.com/ to explore more engines
SEARCHAPI_ENGINE=
-## === Concurrency & Performance Settings ===
+## === Concurrency & Performance Settings ===\n# Number of worker processes per queue - controls parallel job processing\n# Recommended: 4-8 for most setups, adjust based on CPU cores\nNUM_WORKERS_PER_QUEUE=8
# Number of concurrent crawl requests/pages - increase for higher throughput
# Recommended: 8-16 for most setups, 32+ for high-performance servers
CRAWL_CONCURRENT_REQUESTS=10
@@ -43,7 +43,7 @@ RATE_LIMIT_TEST_API_KEY_SCRAPE=
RATE_LIMIT_TEST_API_KEY_CRAWL=
# set if you'd like to use scraping Be to handle JS blocking
SCRAPING_BEE_API_KEY=
-# add for LLM dependednt features (image alt generation, etc.)
+# add for LLM dependent features (image alt generation, etc.)
OPENAI_API_KEY=
BULL_AUTH_KEY=@
# set if you have a llamaparse key you'd like to use to parse pdfs
diff --git a/docker-compose.yaml b/docker-compose.yaml
index 4e275b231..109c989e9 100644
--- a/docker-compose.yaml
+++ b/docker-compose.yaml
@@ -65,14 +65,10 @@ services:
MAX_CONCURRENT_PAGES: ${CRAWL_CONCURRENT_REQUESTS:-10}
networks:
- backend
- deploy:
- resources:
- limits:
- cpus: '2.0'
- memory: 4G
- reservations:
- cpus: '1.0'
- memory: 2G
+ # Resource limits for Docker Compose (not Swarm)
+ cpus: 2.0
+ mem_limit: 4G
+ memswap_limit: 4G
api:
<<: *common-service
@@ -88,14 +84,11 @@ services:
ports:
- "${PORT:-3002}:${INTERNAL_PORT:-3002}"
command: node dist/src/harness.js --start-docker
- deploy:
- resources:
- limits:
- cpus: '4.0' # Increase if you have more CPU cores
- memory: 8G # Increase if you have more RAM
- reservations:
- cpus: '2.0' # Minimum guaranteed CPU
- memory: 4G # Minimum guaranteed RAM
+ # Resource limits for Docker Compose (not Swarm)
+ # Increase if you have more CPU cores/RAM available
+ cpus: 4.0
+ mem_limit: 8G
+ memswap_limit: 8G
redis:
# NOTE: If you want to use Valkey (open source) instead of Redis (source available),