Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(api): excessive robots.txt fetching/parsing (#2562)
tom60a2a89fc01a
diff --git a/apps/api/src/scraper/WebScraper/crawler.ts b/apps/api/src/scraper/WebScraper/crawler.tsindex a1e2e2bf6..91aba1216 100644--- a/apps/api/src/scraper/WebScraper/crawler.ts+++ b/apps/api/src/scraper/WebScraper/crawler.ts@@ -141,6 +141,11 @@ export class WebCrawler {this.location = location;}+ public setBaseUrl(newBase: string): void {+ this.baseUrl = newBase;+ this.robotsTxtUrl = `${this.baseUrl}${this.baseUrl.endsWith("/") ? "" : "/"}robots.txt`;+ }+public async filterLinks(sitemapLinks: string[],limit: number,@@ -411,7 +416,7 @@ export class WebCrawler {this.robotsCrawlDelay = delay !== undefined ? delay : null;const sitemaps = this.robots.getSitemaps();- this.logger.debug("Processed robots.txt", {+ this.logger.debug("Imported robots.txt", {method: "importRobotsTxt",robotsTxtUrl: this.robotsTxtUrl,robotsTxtLength: txt.length,diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.tsindex e1318dd6e..8774eecfe 100644--- a/apps/api/src/scraper/scrapeURL/index.ts+++ b/apps/api/src/scraper/scrapeURL/index.ts@@ -67,6 +67,7 @@ import {createRobotsChecker,isUrlAllowedByRobots,} from "../../lib/robots-txt";+import { getCrawl } from "../../lib/crawl-redis";import {AbortInstance,AbortManager,@@ -886,16 +887,25 @@ export async function scrapeURL(if (!isRobotsTxtPath) {try {- const { content: robotsTxt } = await fetchRobotsTxt(- {- url: urlToCheck,- zeroDataRetention: internalOptions.zeroDataRetention || false,- location: options.location,- },- id,- meta.logger,- meta.abort.asSignal(),- );+ let robotsTxt: string | undefined;+ if (internalOptions.crawlId) {+ const crawl = await getCrawl(internalOptions.crawlId);+ robotsTxt = crawl?.robots;+ }++ if (!robotsTxt) {+ const { content } = await fetchRobotsTxt(+ {+ url: urlToCheck,+ zeroDataRetention: internalOptions.zeroDataRetention || false,+ location: options.location,+ },+ id,+ meta.logger,+ meta.abort.asSignal(),+ );+ robotsTxt = content;+ }const checker = createRobotsChecker(urlToCheck, robotsTxt);const isAllowed = isUrlAllowedByRobots(urlToCheck, checker.robots);diff --git a/apps/api/src/services/worker/scrape-worker.ts b/apps/api/src/services/worker/scrape-worker.tsindex 5b977ac57..776c13852 100644--- a/apps/api/src/services/worker/scrape-worker.ts+++ b/apps/api/src/services/worker/scrape-worker.ts@@ -271,19 +271,26 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {if (job.data.crawl_id) {const sc = (await getCrawl(job.data.crawl_id)) as StoredCrawl;+ let crawler: WebCrawler | null = null;+ if (job.data.crawlerOptions !== null) {+ const teamFlags = (await getACUCTeam(job.data.team_id))?.flags ?? null;+ crawler = crawlToCrawler(+ job.data.crawl_id,+ sc,+ teamFlags,+ sc.originUrl!,+ job.data.crawlerOptions,+ );+ }+if (doc.metadata.url !== undefined &&doc.metadata.sourceURL !== undefined &&normalizeURL(doc.metadata.url, sc) !==normalizeURL(doc.metadata.sourceURL, sc) &&- job.data.crawlerOptions !== null // only on crawls, don't care on batch scrape+ crawler // only on crawls, don't care on batch scrape) {- const crawler = crawlToCrawler(- job.data.crawl_id,- sc,- (await getACUCTeam(job.data.team_id))?.flags ?? null,- );- const filterResult = await crawler.filterURL(+ const filterResult = await crawler!.filterURL(doc.metadata.url,doc.metadata.sourceURL,);@@ -342,14 +349,10 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {}}- if (job.data.crawlerOptions !== null) {+ if (crawler) {if (!sc.cancelled) {- const crawler = crawlToCrawler(- job.data.crawl_id,- sc,- (await getACUCTeam(job.data.team_id))?.flags ?? null,+ crawler.setBaseUrl(doc.metadata.url ?? doc.metadata.sourceURL ?? sc.originUrl!,- job.data.crawlerOptions,);const links = await crawler.filterLinks(