Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Nick: removed 401
Nicolas25dd16bf2a6d
diff --git a/apps/api/src/scraper/WebScraper/single_url.ts b/apps/api/src/scraper/WebScraper/single_url.tsindex 767f30e14..13ca7dd2f 100644--- a/apps/api/src/scraper/WebScraper/single_url.ts+++ b/apps/api/src/scraper/WebScraper/single_url.ts@@ -425,7 +425,7 @@ export async function scrapSingleUrl(Logger.debug(`⛏️ ${scraper}: Successfully scraped ${urlToScrap} with text length >= 100 or screenshot, breaking`);break;}- if (pageStatusCode && (pageStatusCode == 404 || pageStatusCode == 400 || pageStatusCode == 401)) {+ if (pageStatusCode && (pageStatusCode == 404 || pageStatusCode == 400)) {Logger.debug(`⛏️ ${scraper}: Successfully scraped ${urlToScrap} with status code ${pageStatusCode}, breaking`);break;}diff --git a/apps/redis/start-redis-server.sh b/apps/redis/start-redis-server.shindex ed252fde7..30e4a145f 100755--- a/apps/redis/start-redis-server.sh+++ b/apps/redis/start-redis-server.sh@@ -28,3 +28,4 @@ redis-server $PW_ARG \--maxmemory-policy $MAXMEMORY_POLICY \--appendonly $APPENDONLY \--save "$SAVE"+ --save "$SAVE" \