Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
(feat/rtxt) Improved robots control on scrape via flags (#1820)
Nicolasb23535cdce6d
diff --git a/apps/api/src/controllers/v1/scrape.ts b/apps/api/src/controllers/v1/scrape.tsindex f3861a61b..254bc53a9 100644--- a/apps/api/src/controllers/v1/scrape.ts+++ b/apps/api/src/controllers/v1/scrape.ts@@ -74,6 +74,7 @@ export async function scrapeController(unnormalizedSourceURL: preNormalizedBody.url,bypassBilling: isDirectToBullMQ,zeroDataRetention,+ teamFlags: req.acuc?.flags ?? null,},origin,integration: req.body.integration,diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.tsindex 6322f0c16..84ca8d6e7 100644--- a/apps/api/src/controllers/v1/types.ts+++ b/apps/api/src/controllers/v1/types.ts@@ -1048,6 +1048,7 @@ export type TeamFlags = {forceZDR?: boolean;allowZDR?: boolean;zdrCost?: number;+ checkRobotsOnScrape?: boolean;} | null;export type AuthCreditUsageChunkFromTeam = Omit<AuthCreditUsageChunk, "api_key">;diff --git a/apps/api/src/lib/robots-txt.ts b/apps/api/src/lib/robots-txt.tsnew file mode 100644index 000000000..153e8425b--- /dev/null+++ b/apps/api/src/lib/robots-txt.ts@@ -0,0 +1,113 @@+import robotsParser, { Robot } from "robots-parser";+import axios from "axios";+import { axiosTimeout } from "./timeout";+import https from "https";+import { Logger } from "winston";++export interface RobotsTxtChecker {+ robotsTxtUrl: string;+ robotsTxt: string;+ robots: Robot;+}++export async function fetchRobotsTxt(+ url: string,+ skipTlsVerification: boolean = false,+ abort?: AbortSignal,+): Promise<string> {+ const urlObj = new URL(url);+ const robotsTxtUrl = `${urlObj.protocol}//${urlObj.host}/robots.txt`;++ let extraArgs: any = {};+ if (skipTlsVerification) {+ extraArgs.httpsAgent = new https.Agent({+ rejectUnauthorized: false,+ });+ }++ const response = await axios.get(robotsTxtUrl, {+ timeout: axiosTimeout,+ signal: abort,+ ...extraArgs,+ });++ return response.data;+}++export function createRobotsChecker(+ url: string,+ robotsTxt: string,+): RobotsTxtChecker {+ const urlObj = new URL(url);+ const robotsTxtUrl = `${urlObj.protocol}//${urlObj.host}/robots.txt`;+ const robots = robotsParser(robotsTxtUrl, robotsTxt);+ return {+ robotsTxtUrl,+ robotsTxt,+ robots,+ };+}++export function isUrlAllowedByRobots(+ url: string,+ robots: Robot | null,+ userAgents: string[] = ["FireCrawlAgent", "FirecrawlAgent"],+): boolean {+ if (!robots) return true;++ for (const userAgent of userAgents) {+ let isAllowed = robots.isAllowed(url, userAgent);++ // Handle null/undefined responses - default to true (allowed)+ if (isAllowed === null || isAllowed === undefined) {+ isAllowed = true;+ }++ if (isAllowed == null) {+ isAllowed = true;+ }++ // Also check with trailing slash if URL doesn't have one+ // This catches cases like "Disallow: /path/" when user requests "/path"+ if (isAllowed && !url.endsWith("/")) {+ const urlWithSlash = url + "/";+ let isAllowedWithSlash = robots.isAllowed(urlWithSlash, userAgent);++ if (isAllowedWithSlash == null) {+ isAllowedWithSlash = true;+ }++ // If the trailing slash version is explicitly disallowed, block it+ if (isAllowedWithSlash === false) {+ isAllowed = false;+ }+ }++ if (isAllowed) {+ // console.log("isAllowed: true, " + userAgent);+ return true;+ }+ }++ return false;+}++export async function checkRobotsTxt(+ url: string,+ skipTlsVerification: boolean = false,+ logger?: Logger,+ abort?: AbortSignal,+): Promise<boolean> {+ try {+ const robotsTxt = await fetchRobotsTxt(url, skipTlsVerification, abort);+ const checker = createRobotsChecker(url, robotsTxt);+ return isUrlAllowedByRobots(url, checker.robots);+ } catch (error) {+ // If we can't fetch robots.txt, assume it's allowed+ logger?.debug("Failed to fetch robots.txt, allowing scrape", {+ error,+ url,+ });+ return true;+ }+}diff --git a/apps/api/src/main/runWebScraper.ts b/apps/api/src/main/runWebScraper.tsindex 575dbb28b..19167fd99 100644--- a/apps/api/src/main/runWebScraper.ts+++ b/apps/api/src/main/runWebScraper.ts@@ -5,7 +5,7 @@ import {RunWebScraperResult,} from "../types";import { billTeam } from "../services/billing/credit_billing";-import { Document } from "../controllers/v1/types";+import { Document, TeamFlags } from "../controllers/v1/types";import { supabase_service } from "../services/supabase";import { logger as _logger } from "../lib/logger";import { configDotenv } from "dotenv";diff --git a/apps/api/src/scraper/WebScraper/crawler.ts b/apps/api/src/scraper/WebScraper/crawler.tsindex 4b5942e77..37b576577 100644--- a/apps/api/src/scraper/WebScraper/crawler.ts+++ b/apps/api/src/scraper/WebScraper/crawler.ts@@ -11,6 +11,7 @@ import { redisEvictConnection } from "../../services/redis";import { extractLinks } from "../../lib/html-transformer";import { TimeoutSignal } from "../../controllers/v1/types";import { filterLinks } from "../../lib/crawler";+import { fetchRobotsTxt, createRobotsChecker, isUrlAllowedByRobots } from "../../lib/robots-txt";export interface FilterResult {allowed: boolean;@@ -304,23 +305,14 @@ export class WebCrawler {}public async getRobotsTxt(skipTlsVerification = false, abort?: AbortSignal): Promise<string> {- let extraArgs = {};- if (skipTlsVerification) {- extraArgs["httpsAgent"] = new https.Agent({- rejectUnauthorized: false,- });- }- const response = await axios.get(this.robotsTxtUrl, {- timeout: axiosTimeout,- signal: abort,- ...extraArgs,- });- return response.data;+ return fetchRobotsTxt(this.initialUrl, skipTlsVerification, abort);}public importRobotsTxt(txt: string) {this.robotsTxt = txt;- this.robots = robotsParser(this.robotsTxtUrl, this.robotsTxt);+ const checker = createRobotsChecker(this.initialUrl, txt);+ this.robots = checker.robots;+ this.robotsTxtUrl = checker.robotsTxtUrl;const delay = this.robots.getCrawlDelay("FireCrawlAgent") || this.robots.getCrawlDelay("FirecrawlAgent");this.robotsCrawlDelay = delay !== undefined ? delay : null;}@@ -578,9 +570,7 @@ export class WebCrawler {): boolean {return ignoreRobotsTxt? true- : this.robots- ? ((this.robots.isAllowed(url, "FireCrawlAgent") || this.robots.isAllowed(url, "FirecrawlAgent")) ?? true)- : true;+ : isUrlAllowedByRobots(url, this.robots);}private matchesExcludes(url: string, onlyDomains: boolean = false): boolean {diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.tsindex bf55b0870..3dd372cac 100644--- a/apps/api/src/scraper/scrapeURL/index.ts+++ b/apps/api/src/scraper/scrapeURL/index.ts@@ -1,8 +1,8 @@import { Logger } from "winston";import * as Sentry from "@sentry/node";-import { Document, ScrapeOptions, TimeoutSignal } from "../../controllers/v1/types";-import { logger as _logger } from "../../lib/logger";+import { Document, ScrapeOptions, TimeoutSignal, TeamFlags } from "../../controllers/v1/types";+import { logger as _logger, logger } from "../../lib/logger";import {buildFallbackList,Engine,@@ -35,6 +35,7 @@ import { urlSpecificParams } from "./lib/urlSpecificParams";import { loadMock, MockState } from "./lib/mock";import { CostTracking } from "../../lib/extract/extraction-service";import { addIndexRFInsertJob, generateDomainSplits, hashURL, index_supabase_service, normalizeURLForIndex, useIndex } from "../../services/index";+import { checkRobotsTxt } from "../../lib/robots-txt";export type ScrapeUrlResponse = (| {@@ -220,6 +221,7 @@ export type InternalOptions = {saveScrapeResultToGCS?: boolean; // Passed along to fire-enginebypassBilling?: boolean;zeroDataRetention?: boolean;+ teamFlags?: TeamFlags;};export type EngineResultsTracker = {@@ -505,6 +507,31 @@ export async function scrapeURL(meta.logger.info("Rewriting URL");}+ if (internalOptions.teamFlags?.checkRobotsOnScrape) {+ meta.logger.info("Checking robots.txt", {+ checkRobotsOnScrape: internalOptions.teamFlags?.checkRobotsOnScrape,+ url: meta.rewrittenUrl || meta.url,+ });+ const urlToCheck = meta.rewrittenUrl || meta.url;+ const isAllowed = await checkRobotsTxt(+ urlToCheck,+ options.skipTlsVerification,+ meta.logger,+ internalOptions.abort+ );+++ if (!isAllowed) {+ meta.logger.info("URL blocked by robots.txt", { url: urlToCheck });+ return {+ success: false,+ error: new Error("URL blocked by robots.txt"),+ logs: meta.logs,+ engines: meta.results,+ };+ }+ }+const shouldRecordFrequency = useIndex&& meta.options.storeInCache&& !meta.internalOptions.zeroDataRetentiondiff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.tsindex 8991973c4..f900f708d 100644--- a/apps/api/src/services/queue-worker.ts+++ b/apps/api/src/services/queue-worker.ts@@ -1159,6 +1159,7 @@ async function processJob(job: Job & { id: string }, token: string) {}}+const pipeline = await Promise.race([startWebScraperPipeline({job,diff --git a/apps/api/src/types.ts b/apps/api/src/types.tsindex c42720c1c..fb21013e3 100644--- a/apps/api/src/types.ts+++ b/apps/api/src/types.ts@@ -5,6 +5,7 @@ import {ScrapeOptions,Document as V1Document,webhookSchema,+ TeamFlags,} from "./controllers/v1/types";import { ExtractorOptions, Document } from "./lib/entities";import { InternalOptions } from "./scraper/scrapeURL";