Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

(feat/rtxt) Improved robots control on scrape via flags (#1820)

Nicolasb23535cdce6d
diff --git a/apps/api/src/controllers/v1/scrape.ts b/apps/api/src/controllers/v1/scrape.ts
index f3861a61b..254bc53a9 100644
--- a/apps/api/src/controllers/v1/scrape.ts
+++ b/apps/api/src/controllers/v1/scrape.ts
@@ -74,6 +74,7 @@ export async function scrapeController(
unnormalizedSourceURL: preNormalizedBody.url,
bypassBilling: isDirectToBullMQ,
zeroDataRetention,
+ teamFlags: req.acuc?.flags ?? null,
},
origin,
integration: req.body.integration,
diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.ts
index 6322f0c16..84ca8d6e7 100644
--- a/apps/api/src/controllers/v1/types.ts
+++ b/apps/api/src/controllers/v1/types.ts
@@ -1048,6 +1048,7 @@ export type TeamFlags = {
forceZDR?: boolean;
allowZDR?: boolean;
zdrCost?: number;
+ checkRobotsOnScrape?: boolean;
} | null;
export type AuthCreditUsageChunkFromTeam = Omit<AuthCreditUsageChunk, "api_key">;
diff --git a/apps/api/src/lib/robots-txt.ts b/apps/api/src/lib/robots-txt.ts
new file mode 100644
index 000000000..153e8425b
--- /dev/null
+++ b/apps/api/src/lib/robots-txt.ts
@@ -0,0 +1,113 @@
+import robotsParser, { Robot } from "robots-parser";
+import axios from "axios";
+import { axiosTimeout } from "./timeout";
+import https from "https";
+import { Logger } from "winston";
+
+export interface RobotsTxtChecker {
+ robotsTxtUrl: string;
+ robotsTxt: string;
+ robots: Robot;
+}
+
+export async function fetchRobotsTxt(
+ url: string,
+ skipTlsVerification: boolean = false,
+ abort?: AbortSignal,
+): Promise<string> {
+ const urlObj = new URL(url);
+ const robotsTxtUrl = `${urlObj.protocol}//${urlObj.host}/robots.txt`;
+
+ let extraArgs: any = {};
+ if (skipTlsVerification) {
+ extraArgs.httpsAgent = new https.Agent({
+ rejectUnauthorized: false,
+ });
+ }
+
+ const response = await axios.get(robotsTxtUrl, {
+ timeout: axiosTimeout,
+ signal: abort,
+ ...extraArgs,
+ });
+
+ return response.data;
+}
+
+export function createRobotsChecker(
+ url: string,
+ robotsTxt: string,
+): RobotsTxtChecker {
+ const urlObj = new URL(url);
+ const robotsTxtUrl = `${urlObj.protocol}//${urlObj.host}/robots.txt`;
+ const robots = robotsParser(robotsTxtUrl, robotsTxt);
+ return {
+ robotsTxtUrl,
+ robotsTxt,
+ robots,
+ };
+}
+
+export function isUrlAllowedByRobots(
+ url: string,
+ robots: Robot | null,
+ userAgents: string[] = ["FireCrawlAgent", "FirecrawlAgent"],
+): boolean {
+ if (!robots) return true;
+
+ for (const userAgent of userAgents) {
+ let isAllowed = robots.isAllowed(url, userAgent);
+
+ // Handle null/undefined responses - default to true (allowed)
+ if (isAllowed === null || isAllowed === undefined) {
+ isAllowed = true;
+ }
+
+ if (isAllowed == null) {
+ isAllowed = true;
+ }
+
+ // Also check with trailing slash if URL doesn't have one
+ // This catches cases like "Disallow: /path/" when user requests "/path"
+ if (isAllowed && !url.endsWith("/")) {
+ const urlWithSlash = url + "/";
+ let isAllowedWithSlash = robots.isAllowed(urlWithSlash, userAgent);
+
+ if (isAllowedWithSlash == null) {
+ isAllowedWithSlash = true;
+ }
+
+ // If the trailing slash version is explicitly disallowed, block it
+ if (isAllowedWithSlash === false) {
+ isAllowed = false;
+ }
+ }
+
+ if (isAllowed) {
+ // console.log("isAllowed: true, " + userAgent);
+ return true;
+ }
+ }
+
+ return false;
+}
+
+export async function checkRobotsTxt(
+ url: string,
+ skipTlsVerification: boolean = false,
+ logger?: Logger,
+ abort?: AbortSignal,
+): Promise<boolean> {
+ try {
+ const robotsTxt = await fetchRobotsTxt(url, skipTlsVerification, abort);
+ const checker = createRobotsChecker(url, robotsTxt);
+ return isUrlAllowedByRobots(url, checker.robots);
+ } catch (error) {
+ // If we can't fetch robots.txt, assume it's allowed
+ logger?.debug("Failed to fetch robots.txt, allowing scrape", {
+ error,
+ url,
+ });
+ return true;
+ }
+}
diff --git a/apps/api/src/main/runWebScraper.ts b/apps/api/src/main/runWebScraper.ts
index 575dbb28b..19167fd99 100644
--- a/apps/api/src/main/runWebScraper.ts
+++ b/apps/api/src/main/runWebScraper.ts
@@ -5,7 +5,7 @@ import {
RunWebScraperResult,
} from "../types";
import { billTeam } from "../services/billing/credit_billing";
-import { Document } from "../controllers/v1/types";
+import { Document, TeamFlags } from "../controllers/v1/types";
import { supabase_service } from "../services/supabase";
import { logger as _logger } from "../lib/logger";
import { configDotenv } from "dotenv";
diff --git a/apps/api/src/scraper/WebScraper/crawler.ts b/apps/api/src/scraper/WebScraper/crawler.ts
index 4b5942e77..37b576577 100644
--- a/apps/api/src/scraper/WebScraper/crawler.ts
+++ b/apps/api/src/scraper/WebScraper/crawler.ts
@@ -11,6 +11,7 @@ import { redisEvictConnection } from "../../services/redis";
import { extractLinks } from "../../lib/html-transformer";
import { TimeoutSignal } from "../../controllers/v1/types";
import { filterLinks } from "../../lib/crawler";
+import { fetchRobotsTxt, createRobotsChecker, isUrlAllowedByRobots } from "../../lib/robots-txt";
export interface FilterResult {
allowed: boolean;
@@ -304,23 +305,14 @@ export class WebCrawler {
}
public async getRobotsTxt(skipTlsVerification = false, abort?: AbortSignal): Promise<string> {
- let extraArgs = {};
- if (skipTlsVerification) {
- extraArgs["httpsAgent"] = new https.Agent({
- rejectUnauthorized: false,
- });
- }
- const response = await axios.get(this.robotsTxtUrl, {
- timeout: axiosTimeout,
- signal: abort,
- ...extraArgs,
- });
- return response.data;
+ return fetchRobotsTxt(this.initialUrl, skipTlsVerification, abort);
}
public importRobotsTxt(txt: string) {
this.robotsTxt = txt;
- this.robots = robotsParser(this.robotsTxtUrl, this.robotsTxt);
+ const checker = createRobotsChecker(this.initialUrl, txt);
+ this.robots = checker.robots;
+ this.robotsTxtUrl = checker.robotsTxtUrl;
const delay = this.robots.getCrawlDelay("FireCrawlAgent") || this.robots.getCrawlDelay("FirecrawlAgent");
this.robotsCrawlDelay = delay !== undefined ? delay : null;
}
@@ -578,9 +570,7 @@ export class WebCrawler {
): boolean {
return ignoreRobotsTxt
? true
- : this.robots
- ? ((this.robots.isAllowed(url, "FireCrawlAgent") || this.robots.isAllowed(url, "FirecrawlAgent")) ?? true)
- : true;
+ : isUrlAllowedByRobots(url, this.robots);
}
private matchesExcludes(url: string, onlyDomains: boolean = false): boolean {
diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.ts
index bf55b0870..3dd372cac 100644
--- a/apps/api/src/scraper/scrapeURL/index.ts
+++ b/apps/api/src/scraper/scrapeURL/index.ts
@@ -1,8 +1,8 @@
import { Logger } from "winston";
import * as Sentry from "@sentry/node";
-import { Document, ScrapeOptions, TimeoutSignal } from "../../controllers/v1/types";
-import { logger as _logger } from "../../lib/logger";
+import { Document, ScrapeOptions, TimeoutSignal, TeamFlags } from "../../controllers/v1/types";
+import { logger as _logger, logger } from "../../lib/logger";
import {
buildFallbackList,
Engine,
@@ -35,6 +35,7 @@ import { urlSpecificParams } from "./lib/urlSpecificParams";
import { loadMock, MockState } from "./lib/mock";
import { CostTracking } from "../../lib/extract/extraction-service";
import { addIndexRFInsertJob, generateDomainSplits, hashURL, index_supabase_service, normalizeURLForIndex, useIndex } from "../../services/index";
+import { checkRobotsTxt } from "../../lib/robots-txt";
export type ScrapeUrlResponse = (
| {
@@ -220,6 +221,7 @@ export type InternalOptions = {
saveScrapeResultToGCS?: boolean; // Passed along to fire-engine
bypassBilling?: boolean;
zeroDataRetention?: boolean;
+ teamFlags?: TeamFlags;
};
export type EngineResultsTracker = {
@@ -505,6 +507,31 @@ export async function scrapeURL(
meta.logger.info("Rewriting URL");
}
+ if (internalOptions.teamFlags?.checkRobotsOnScrape) {
+ meta.logger.info("Checking robots.txt", {
+ checkRobotsOnScrape: internalOptions.teamFlags?.checkRobotsOnScrape,
+ url: meta.rewrittenUrl || meta.url,
+ });
+ const urlToCheck = meta.rewrittenUrl || meta.url;
+ const isAllowed = await checkRobotsTxt(
+ urlToCheck,
+ options.skipTlsVerification,
+ meta.logger,
+ internalOptions.abort
+ );
+
+
+ if (!isAllowed) {
+ meta.logger.info("URL blocked by robots.txt", { url: urlToCheck });
+ return {
+ success: false,
+ error: new Error("URL blocked by robots.txt"),
+ logs: meta.logs,
+ engines: meta.results,
+ };
+ }
+ }
+
const shouldRecordFrequency = useIndex
&& meta.options.storeInCache
&& !meta.internalOptions.zeroDataRetention
diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.ts
index 8991973c4..f900f708d 100644
--- a/apps/api/src/services/queue-worker.ts
+++ b/apps/api/src/services/queue-worker.ts
@@ -1159,6 +1159,7 @@ async function processJob(job: Job & { id: string }, token: string) {
}
}
+
const pipeline = await Promise.race([
startWebScraperPipeline({
job,
diff --git a/apps/api/src/types.ts b/apps/api/src/types.ts
index c42720c1c..fb21013e3 100644
--- a/apps/api/src/types.ts
+++ b/apps/api/src/types.ts
@@ -5,6 +5,7 @@ import {
ScrapeOptions,
Document as V1Document,
webhookSchema,
+ TeamFlags,
} from "./controllers/v1/types";
import { ExtractorOptions, Document } from "./lib/entities";
import { InternalOptions } from "./scraper/scrapeURL";