Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fixes docker-compose and 401 error

rafaelsideguided316d52c963e
diff --git a/apps/api/src/scraper/WebScraper/single_url.ts b/apps/api/src/scraper/WebScraper/single_url.ts
index 767f30e14..13ca7dd2f 100644
--- a/apps/api/src/scraper/WebScraper/single_url.ts
+++ b/apps/api/src/scraper/WebScraper/single_url.ts
@@ -425,7 +425,7 @@ export async function scrapSingleUrl(
Logger.debug(`⛏️ ${scraper}: Successfully scraped ${urlToScrap} with text length >= 100 or screenshot, breaking`);
break;
}
- if (pageStatusCode && (pageStatusCode == 404 || pageStatusCode == 400 || pageStatusCode == 401)) {
+ if (pageStatusCode && (pageStatusCode == 404 || pageStatusCode == 400)) {
Logger.debug(`⛏️ ${scraper}: Successfully scraped ${urlToScrap} with status code ${pageStatusCode}, breaking`);
break;
}
diff --git a/docker-compose.yaml b/docker-compose.yaml
index da1265c38..24c007c2c 100644
--- a/docker-compose.yaml
+++ b/docker-compose.yaml
@@ -4,30 +4,6 @@ x-common-service: &common-service
build: apps/api
networks:
- backend
- environment:
- - REDIS_URL=${REDIS_URL:-redis://redis:6379}
- - REDIS_RATE_LIMIT_URL=${REDIS_URL:-redis://redis:6379}
- - PLAYWRIGHT_MICROSERVICE_URL=${PLAYWRIGHT_MICROSERVICE_URL:-http://playwright-service:3000}
- - USE_DB_AUTHENTICATION=${USE_DB_AUTHENTICATION}
- - PORT=${PORT:-3002}
- - NUM_WORKERS_PER_QUEUE=${NUM_WORKERS_PER_QUEUE}
- - OPENAI_API_KEY=${OPENAI_API_KEY}
- - OPENAI_BASE_URL=${OPENAI_BASE_URL}
- - MODEL_NAME=${MODEL_NAME:-gpt-4o}
- - SLACK_WEBHOOK_URL=${SLACK_WEBHOOK_URL}
- - LLAMAPARSE_API_KEY=${LLAMAPARSE_API_KEY}
- - LOGTAIL_KEY=${LOGTAIL_KEY}
- - BULL_AUTH_KEY=${BULL_AUTH_KEY}
- - TEST_API_KEY=${TEST_API_KEY}
- - POSTHOG_API_KEY=${POSTHOG_API_KEY}
- - POSTHOG_HOST=${POSTHOG_HOST}
- - SUPABASE_ANON_TOKEN=${SUPABASE_ANON_TOKEN}
- - SUPABASE_URL=${SUPABASE_URL}
- - SUPABASE_SERVICE_TOKEN=${SUPABASE_SERVICE_TOKEN}
- - SCRAPING_BEE_API_KEY=${SCRAPING_BEE_API_KEY}
- - HOST=${HOST:-0.0.0.0}
- - SELF_HOSTED_WEBHOOK_URL=${SELF_HOSTED_WEBHOOK_URL}
- - LOGGING_LEVEL=${LOGGING_LEVEL}
extra_hosts:
- "host.docker.internal:host-gateway"
@@ -46,7 +22,29 @@ services:
api:
<<: *common-service
environment:
- <<: *common-service.environment
+ REDIS_URL: ${REDIS_URL:-redis://redis:6379}
+ REDIS_RATE_LIMIT_URL: ${REDIS_URL:-redis://redis:6379}
+ PLAYWRIGHT_MICROSERVICE_URL: ${PLAYWRIGHT_MICROSERVICE_URL:-http://playwright-service:3000}
+ USE_DB_AUTHENTICATION: ${USE_DB_AUTHENTICATION}
+ PORT: ${PORT:-3002}
+ NUM_WORKERS_PER_QUEUE: ${NUM_WORKERS_PER_QUEUE}
+ OPENAI_API_KEY: ${OPENAI_API_KEY}
+ OPENAI_BASE_URL: ${OPENAI_BASE_URL}
+ MODEL_NAME: ${MODEL_NAME:-gpt-4o}
+ SLACK_WEBHOOK_URL: ${SLACK_WEBHOOK_URL}
+ LLAMAPARSE_API_KEY: ${LLAMAPARSE_API_KEY}
+ LOGTAIL_KEY: ${LOGTAIL_KEY}
+ BULL_AUTH_KEY: ${BULL_AUTH_KEY}
+ TEST_API_KEY: ${TEST_API_KEY}
+ POSTHOG_API_KEY: ${POSTHOG_API_KEY}
+ POSTHOG_HOST: ${POSTHOG_HOST}
+ SUPABASE_ANON_TOKEN: ${SUPABASE_ANON_TOKEN}
+ SUPABASE_URL: ${SUPABASE_URL}
+ SUPABASE_SERVICE_TOKEN: ${SUPABASE_SERVICE_TOKEN}
+ SCRAPING_BEE_API_KEY: ${SCRAPING_BEE_API_KEY}
+ HOST: ${HOST:-0.0.0.0}
+ SELF_HOSTED_WEBHOOK_URL: ${SELF_HOSTED_WEBHOOK_URL}
+ LOGGING_LEVEL: ${LOGGING_LEVEL}
FLY_PROCESS_GROUP: app
depends_on:
- redis
@@ -58,7 +56,29 @@ services:
worker:
<<: *common-service
environment:
- <<: *common-service.environment
+ REDIS_URL: ${REDIS_URL:-redis://redis:6379}
+ REDIS_RATE_LIMIT_URL: ${REDIS_URL:-redis://redis:6379}
+ PLAYWRIGHT_MICROSERVICE_URL: ${PLAYWRIGHT_MICROSERVICE_URL:-http://playwright-service:3000}
+ USE_DB_AUTHENTICATION: ${USE_DB_AUTHENTICATION}
+ PORT: ${PORT:-3002}
+ NUM_WORKERS_PER_QUEUE: ${NUM_WORKERS_PER_QUEUE}
+ OPENAI_API_KEY: ${OPENAI_API_KEY}
+ OPENAI_BASE_URL: ${OPENAI_BASE_URL}
+ MODEL_NAME: ${MODEL_NAME:-gpt-4o}
+ SLACK_WEBHOOK_URL: ${SLACK_WEBHOOK_URL}
+ LLAMAPARSE_API_KEY: ${LLAMAPARSE_API_KEY}
+ LOGTAIL_KEY: ${LOGTAIL_KEY}
+ BULL_AUTH_KEY: ${BULL_AUTH_KEY}
+ TEST_API_KEY: ${TEST_API_KEY}
+ POSTHOG_API_KEY: ${POSTHOG_API_KEY}
+ POSTHOG_HOST: ${POSTHOG_HOST}
+ SUPABASE_ANON_TOKEN: ${SUPABASE_ANON_TOKEN}
+ SUPABASE_URL: ${SUPABASE_URL}
+ SUPABASE_SERVICE_TOKEN: ${SUPABASE_SERVICE_TOKEN}
+ SCRAPING_BEE_API_KEY: ${SCRAPING_BEE_API_KEY}
+ HOST: ${HOST:-0.0.0.0}
+ SELF_HOSTED_WEBHOOK_URL: ${SELF_HOSTED_WEBHOOK_URL}
+ LOGGING_LEVEL: ${LOGGING_LEVEL}
FLY_PROCESS_GROUP: worker
depends_on:
- redis