Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix/check files on crawl
rafaelsideguide180801225b50
diff --git a/apps/api/src/scraper/WebScraper/crawler.ts b/apps/api/src/scraper/WebScraper/crawler.tsindex d5dadaf81..009a5933d 100644--- a/apps/api/src/scraper/WebScraper/crawler.ts+++ b/apps/api/src/scraper/WebScraper/crawler.ts@@ -136,6 +136,10 @@ export class WebCrawler {return false;}+ if (this.isFile(link)) {+ return false;+ }+return true;}).slice(0, limit);@@ -478,7 +482,14 @@ export class WebCrawler {".webp",".inc"];- return fileExtensions.some((ext) => url.toLowerCase().endsWith(ext));++ try {+ const urlWithoutQuery = url.split('?')[0].toLowerCase();+ return fileExtensions.some((ext) => urlWithoutQuery.endsWith(ext));+ } catch (error) {+ Logger.error(`Error processing URL in isFile: ${error}`);+ return false;+ }}private isSocialMediaOrEmail(url: string): boolean {