Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(crawler): recognize sitemaps in robots.txt
Gergő Móricz23bb172592b4
diff --git a/apps/api/src/controllers/v1/map.ts b/apps/api/src/controllers/v1/map.tsindex 2afae0d4d..c8f7dd96c 100644--- a/apps/api/src/controllers/v1/map.ts+++ b/apps/api/src/controllers/v1/map.ts@@ -85,6 +85,11 @@ export async function getMapResults({const crawler = crawlToCrawler(id, sc);+ try {+ sc.robots = await crawler.getRobotsTxt();+ await crawler.importRobotsTxt(sc.robots);+ } catch (_) {}+// If sitemapOnly is true, only get links from sitemapif (crawlerOptions.sitemapOnly) {const sitemap = await crawler.tryGetSitemap(diff --git a/apps/api/src/scraper/WebScraper/crawler.ts b/apps/api/src/scraper/WebScraper/crawler.tsindex 5662fff9f..1cca7a185 100644--- a/apps/api/src/scraper/WebScraper/crawler.ts+++ b/apps/api/src/scraper/WebScraper/crawler.ts@@ -2,7 +2,7 @@ import axios, { AxiosError } from "axios";import cheerio, { load } from "cheerio";import { URL } from "url";import { getLinksFromSitemap } from "./sitemap";-import robotsParser from "robots-parser";+import robotsParser, { Robot } from "robots-parser";import { getURLDepth } from "./utils/maxDepthUtils";import { axiosTimeout } from "../../lib/timeout";import { logger as _logger } from "../../lib/logger";@@ -20,7 +20,7 @@ export class WebCrawler {private crawledUrls: Map<string, string> = new Map();private limit: number;private robotsTxtUrl: string;- public robots: any;+ public robots: Robot;private generateImgAltText: boolean;private allowBackwardCrawling: boolean;private allowExternalContentLinks: boolean;@@ -63,7 +63,7 @@ export class WebCrawler {this.includes = Array.isArray(includes) ? includes : [];this.excludes = Array.isArray(excludes) ? excludes : [];this.limit = limit;- this.robotsTxtUrl = `${this.baseUrl}/robots.txt`;+ this.robotsTxtUrl = `${this.baseUrl}${this.baseUrl.endsWith("/") ? "" : "/"}robots.txt`;this.robots = robotsParser(this.robotsTxtUrl, "");// Deprecated, use limit insteadthis.maxCrawledLinks = maxCrawledLinks ?? limit;@@ -217,45 +217,46 @@ export class WebCrawler {};const _urlsHandler = async (urls: string[]) => {- let uniqueURLs: string[] = [];- for (const url of urls) {- if (- await redisConnection.sadd(- "sitemap:" + this.jobId + ":links",- normalizeUrl(url),- )- ) {- uniqueURLs.push(url);+ if (fromMap && onlySitemap) {+ return urlsHandler(urls);+ } else {+ let filteredLinks = this.filterLinks(+ [...new Set(urls)],+ leftOfLimit,+ this.maxCrawledDepth,+ fromMap,+ );+ leftOfLimit -= filteredLinks.length;+ let uniqueURLs: string[] = [];+ for (const url of urls) {+ if (+ await redisConnection.sadd(+ "sitemap:" + this.jobId + ":links",+ normalizeUrl(url),+ )+ ) {+ uniqueURLs.push(url);+ }}- }- await redisConnection.expire(- "sitemap:" + this.jobId + ":links",- 3600,- "NX",- );- if (uniqueURLs.length > 0) {- urlsHandler(uniqueURLs);+ await redisConnection.expire(+ "sitemap:" + this.jobId + ":links",+ 3600,+ "NX",+ );+ if (uniqueURLs.length > 0) {+ return urlsHandler(uniqueURLs);+ }}};- let count = await this.tryFetchSitemapLinks(- this.initialUrl,- (urls: string[]) => {- if (fromMap && onlySitemap) {- return urlsHandler(urls);- } else {- let filteredLinks = this.filterLinks(- [...new Set(urls)],- leftOfLimit,- this.maxCrawledDepth,- fromMap,- );- leftOfLimit -= filteredLinks.length;- return _urlsHandler(filteredLinks);- }- },- );+ let count = (await Promise.all([+ this.tryFetchSitemapLinks(+ this.initialUrl,+ _urlsHandler,+ ),+ ...this.robots.getSitemaps().map(x => this.tryFetchSitemapLinks(x, _urlsHandler)),+ ])).reduce((a,x) => a+x, 0);if (count > 0) {if (