Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fixes docker-compose and 401 error
rafaelsideguided316d52c963e
diff --git a/apps/api/src/scraper/WebScraper/single_url.ts b/apps/api/src/scraper/WebScraper/single_url.tsindex 767f30e14..13ca7dd2f 100644--- a/apps/api/src/scraper/WebScraper/single_url.ts+++ b/apps/api/src/scraper/WebScraper/single_url.ts@@ -425,7 +425,7 @@ export async function scrapSingleUrl(Logger.debug(`⛏️ ${scraper}: Successfully scraped ${urlToScrap} with text length >= 100 or screenshot, breaking`);break;}- if (pageStatusCode && (pageStatusCode == 404 || pageStatusCode == 400 || pageStatusCode == 401)) {+ if (pageStatusCode && (pageStatusCode == 404 || pageStatusCode == 400)) {Logger.debug(`⛏️ ${scraper}: Successfully scraped ${urlToScrap} with status code ${pageStatusCode}, breaking`);break;}diff --git a/docker-compose.yaml b/docker-compose.yamlindex da1265c38..24c007c2c 100644--- a/docker-compose.yaml+++ b/docker-compose.yaml@@ -4,30 +4,6 @@ x-common-service: &common-servicebuild: apps/apinetworks:- backend- environment:- - REDIS_URL=${REDIS_URL:-redis://redis:6379}- - REDIS_RATE_LIMIT_URL=${REDIS_URL:-redis://redis:6379}- - PLAYWRIGHT_MICROSERVICE_URL=${PLAYWRIGHT_MICROSERVICE_URL:-http://playwright-service:3000}- - USE_DB_AUTHENTICATION=${USE_DB_AUTHENTICATION}- - PORT=${PORT:-3002}- - NUM_WORKERS_PER_QUEUE=${NUM_WORKERS_PER_QUEUE}- - OPENAI_API_KEY=${OPENAI_API_KEY}- - OPENAI_BASE_URL=${OPENAI_BASE_URL}- - MODEL_NAME=${MODEL_NAME:-gpt-4o}- - SLACK_WEBHOOK_URL=${SLACK_WEBHOOK_URL}- - LLAMAPARSE_API_KEY=${LLAMAPARSE_API_KEY}- - LOGTAIL_KEY=${LOGTAIL_KEY}- - BULL_AUTH_KEY=${BULL_AUTH_KEY}- - TEST_API_KEY=${TEST_API_KEY}- - POSTHOG_API_KEY=${POSTHOG_API_KEY}- - POSTHOG_HOST=${POSTHOG_HOST}- - SUPABASE_ANON_TOKEN=${SUPABASE_ANON_TOKEN}- - SUPABASE_URL=${SUPABASE_URL}- - SUPABASE_SERVICE_TOKEN=${SUPABASE_SERVICE_TOKEN}- - SCRAPING_BEE_API_KEY=${SCRAPING_BEE_API_KEY}- - HOST=${HOST:-0.0.0.0}- - SELF_HOSTED_WEBHOOK_URL=${SELF_HOSTED_WEBHOOK_URL}- - LOGGING_LEVEL=${LOGGING_LEVEL}extra_hosts:- "host.docker.internal:host-gateway"@@ -46,7 +22,29 @@ services:api:<<: *common-serviceenvironment:- <<: *common-service.environment+ REDIS_URL: ${REDIS_URL:-redis://redis:6379}+ REDIS_RATE_LIMIT_URL: ${REDIS_URL:-redis://redis:6379}+ PLAYWRIGHT_MICROSERVICE_URL: ${PLAYWRIGHT_MICROSERVICE_URL:-http://playwright-service:3000}+ USE_DB_AUTHENTICATION: ${USE_DB_AUTHENTICATION}+ PORT: ${PORT:-3002}+ NUM_WORKERS_PER_QUEUE: ${NUM_WORKERS_PER_QUEUE}+ OPENAI_API_KEY: ${OPENAI_API_KEY}+ OPENAI_BASE_URL: ${OPENAI_BASE_URL}+ MODEL_NAME: ${MODEL_NAME:-gpt-4o}+ SLACK_WEBHOOK_URL: ${SLACK_WEBHOOK_URL}+ LLAMAPARSE_API_KEY: ${LLAMAPARSE_API_KEY}+ LOGTAIL_KEY: ${LOGTAIL_KEY}+ BULL_AUTH_KEY: ${BULL_AUTH_KEY}+ TEST_API_KEY: ${TEST_API_KEY}+ POSTHOG_API_KEY: ${POSTHOG_API_KEY}+ POSTHOG_HOST: ${POSTHOG_HOST}+ SUPABASE_ANON_TOKEN: ${SUPABASE_ANON_TOKEN}+ SUPABASE_URL: ${SUPABASE_URL}+ SUPABASE_SERVICE_TOKEN: ${SUPABASE_SERVICE_TOKEN}+ SCRAPING_BEE_API_KEY: ${SCRAPING_BEE_API_KEY}+ HOST: ${HOST:-0.0.0.0}+ SELF_HOSTED_WEBHOOK_URL: ${SELF_HOSTED_WEBHOOK_URL}+ LOGGING_LEVEL: ${LOGGING_LEVEL}FLY_PROCESS_GROUP: appdepends_on:- redis@@ -58,7 +56,29 @@ services:worker:<<: *common-serviceenvironment:- <<: *common-service.environment+ REDIS_URL: ${REDIS_URL:-redis://redis:6379}+ REDIS_RATE_LIMIT_URL: ${REDIS_URL:-redis://redis:6379}+ PLAYWRIGHT_MICROSERVICE_URL: ${PLAYWRIGHT_MICROSERVICE_URL:-http://playwright-service:3000}+ USE_DB_AUTHENTICATION: ${USE_DB_AUTHENTICATION}+ PORT: ${PORT:-3002}+ NUM_WORKERS_PER_QUEUE: ${NUM_WORKERS_PER_QUEUE}+ OPENAI_API_KEY: ${OPENAI_API_KEY}+ OPENAI_BASE_URL: ${OPENAI_BASE_URL}+ MODEL_NAME: ${MODEL_NAME:-gpt-4o}+ SLACK_WEBHOOK_URL: ${SLACK_WEBHOOK_URL}+ LLAMAPARSE_API_KEY: ${LLAMAPARSE_API_KEY}+ LOGTAIL_KEY: ${LOGTAIL_KEY}+ BULL_AUTH_KEY: ${BULL_AUTH_KEY}+ TEST_API_KEY: ${TEST_API_KEY}+ POSTHOG_API_KEY: ${POSTHOG_API_KEY}+ POSTHOG_HOST: ${POSTHOG_HOST}+ SUPABASE_ANON_TOKEN: ${SUPABASE_ANON_TOKEN}+ SUPABASE_URL: ${SUPABASE_URL}+ SUPABASE_SERVICE_TOKEN: ${SUPABASE_SERVICE_TOKEN}+ SCRAPING_BEE_API_KEY: ${SCRAPING_BEE_API_KEY}+ HOST: ${HOST:-0.0.0.0}+ SELF_HOSTED_WEBHOOK_URL: ${SELF_HOSTED_WEBHOOK_URL}+ LOGGING_LEVEL: ${LOGGING_LEVEL}FLY_PROCESS_GROUP: workerdepends_on:- redis