Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(api): excessive robots.txt fetching/parsing (#2562)

tom60a2a89fc01a
diff --git a/apps/api/src/scraper/WebScraper/crawler.ts b/apps/api/src/scraper/WebScraper/crawler.ts
index a1e2e2bf6..91aba1216 100644
--- a/apps/api/src/scraper/WebScraper/crawler.ts
+++ b/apps/api/src/scraper/WebScraper/crawler.ts
@@ -141,6 +141,11 @@ export class WebCrawler {
this.location = location;
}
+ public setBaseUrl(newBase: string): void {
+ this.baseUrl = newBase;
+ this.robotsTxtUrl = `${this.baseUrl}${this.baseUrl.endsWith("/") ? "" : "/"}robots.txt`;
+ }
+
public async filterLinks(
sitemapLinks: string[],
limit: number,
@@ -411,7 +416,7 @@ export class WebCrawler {
this.robotsCrawlDelay = delay !== undefined ? delay : null;
const sitemaps = this.robots.getSitemaps();
- this.logger.debug("Processed robots.txt", {
+ this.logger.debug("Imported robots.txt", {
method: "importRobotsTxt",
robotsTxtUrl: this.robotsTxtUrl,
robotsTxtLength: txt.length,
diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.ts
index e1318dd6e..8774eecfe 100644
--- a/apps/api/src/scraper/scrapeURL/index.ts
+++ b/apps/api/src/scraper/scrapeURL/index.ts
@@ -67,6 +67,7 @@ import {
createRobotsChecker,
isUrlAllowedByRobots,
} from "../../lib/robots-txt";
+import { getCrawl } from "../../lib/crawl-redis";
import {
AbortInstance,
AbortManager,
@@ -886,16 +887,25 @@ export async function scrapeURL(
if (!isRobotsTxtPath) {
try {
- const { content: robotsTxt } = await fetchRobotsTxt(
- {
- url: urlToCheck,
- zeroDataRetention: internalOptions.zeroDataRetention || false,
- location: options.location,
- },
- id,
- meta.logger,
- meta.abort.asSignal(),
- );
+ let robotsTxt: string | undefined;
+ if (internalOptions.crawlId) {
+ const crawl = await getCrawl(internalOptions.crawlId);
+ robotsTxt = crawl?.robots;
+ }
+
+ if (!robotsTxt) {
+ const { content } = await fetchRobotsTxt(
+ {
+ url: urlToCheck,
+ zeroDataRetention: internalOptions.zeroDataRetention || false,
+ location: options.location,
+ },
+ id,
+ meta.logger,
+ meta.abort.asSignal(),
+ );
+ robotsTxt = content;
+ }
const checker = createRobotsChecker(urlToCheck, robotsTxt);
const isAllowed = isUrlAllowedByRobots(urlToCheck, checker.robots);
diff --git a/apps/api/src/services/worker/scrape-worker.ts b/apps/api/src/services/worker/scrape-worker.ts
index 5b977ac57..776c13852 100644
--- a/apps/api/src/services/worker/scrape-worker.ts
+++ b/apps/api/src/services/worker/scrape-worker.ts
@@ -271,19 +271,26 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {
if (job.data.crawl_id) {
const sc = (await getCrawl(job.data.crawl_id)) as StoredCrawl;
+ let crawler: WebCrawler | null = null;
+ if (job.data.crawlerOptions !== null) {
+ const teamFlags = (await getACUCTeam(job.data.team_id))?.flags ?? null;
+ crawler = crawlToCrawler(
+ job.data.crawl_id,
+ sc,
+ teamFlags,
+ sc.originUrl!,
+ job.data.crawlerOptions,
+ );
+ }
+
if (
doc.metadata.url !== undefined &&
doc.metadata.sourceURL !== undefined &&
normalizeURL(doc.metadata.url, sc) !==
normalizeURL(doc.metadata.sourceURL, sc) &&
- job.data.crawlerOptions !== null // only on crawls, don't care on batch scrape
+ crawler // only on crawls, don't care on batch scrape
) {
- const crawler = crawlToCrawler(
- job.data.crawl_id,
- sc,
- (await getACUCTeam(job.data.team_id))?.flags ?? null,
- );
- const filterResult = await crawler.filterURL(
+ const filterResult = await crawler!.filterURL(
doc.metadata.url,
doc.metadata.sourceURL,
);
@@ -342,14 +349,10 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {
}
}
- if (job.data.crawlerOptions !== null) {
+ if (crawler) {
if (!sc.cancelled) {
- const crawler = crawlToCrawler(
- job.data.crawl_id,
- sc,
- (await getACUCTeam(job.data.team_id))?.flags ?? null,
+ crawler.setBaseUrl(
doc.metadata.url ?? doc.metadata.sourceURL ?? sc.originUrl!,
- job.data.crawlerOptions,
);
const links = await crawler.filterLinks(