Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(api): record robots.txt denials on crawls so the status warning and robotsBlocked work (#4900)

claude[bot]978880529524
diff --git a/apps/api/src/__tests__/snips/v2/crawl-robots.test.ts b/apps/api/src/__tests__/snips/v2/crawl-robots.test.ts
new file mode 100644
index 000000000..cb1659ede
--- /dev/null
+++ b/apps/api/src/__tests__/snips/v2/crawl-robots.test.ts
@@ -0,0 +1,82 @@
+import { describe, expect } from "vitest";
+import {
+ ALLOW_TEST_SUITE_WEBSITE,
+ concurrentIf,
+ TEST_SUITE_WEBSITE,
+} from "../lib";
+import { crawl, crawlErrors, Identity, idmux, scrapeTimeout } from "./lib";
+
+// The test site's robots.txt disallows /robots-test/blocked/. The
+// /robots-test/ page is allowed and links to the blocked page.
+const ROBOTS_TEST_URL = TEST_SUITE_WEBSITE + "/robots-test/";
+const ROBOTS_BLOCKED_URL = TEST_SUITE_WEBSITE + "/robots-test/blocked/";
+
+let identity: Identity;
+
+beforeAll(async () => {
+ identity = await idmux({
+ name: "crawl-robots",
+ concurrency: 20,
+ credits: 1000000,
+ });
+}, 10000);
+
+const isBlockedUrl = (url: string) => url.includes("/robots-test/blocked");
+
+describe("Crawl robots.txt reporting", () => {
+ concurrentIf(ALLOW_TEST_SUITE_WEBSITE)(
+ "reports a start URL that robots.txt disallows",
+ async () => {
+ const results = await crawl(
+ { url: ROBOTS_BLOCKED_URL, limit: 5 },
+ identity,
+ false,
+ );
+
+ expect(results.status).not.toBe("scraping");
+ expect(results.warning ?? "").toContain("robots.txt");
+ expect(results.warning ?? "").toContain("robotsBlocked");
+ expect(
+ (await crawlErrors(results.id, identity)).robotsBlocked.some(
+ isBlockedUrl,
+ ),
+ ).toBe(true);
+ },
+ 5 * scrapeTimeout,
+ );
+
+ concurrentIf(ALLOW_TEST_SUITE_WEBSITE)(
+ "reports a discovered link that robots.txt disallows",
+ async () => {
+ const results = await crawl({ url: ROBOTS_TEST_URL, limit: 5 }, identity);
+
+ expect(results.data.some(d => isBlockedUrl(d.metadata?.url ?? ""))).toBe(
+ false,
+ );
+ expect(results.warning).toContain("robots.txt");
+ expect(results.warning).toContain("robotsBlocked");
+ expect(
+ (await crawlErrors(results.id, identity)).robotsBlocked.some(
+ isBlockedUrl,
+ ),
+ ).toBe(true);
+ },
+ 5 * scrapeTimeout,
+ );
+
+ concurrentIf(ALLOW_TEST_SUITE_WEBSITE)(
+ "does not warn when robots.txt allows every crawled page",
+ async () => {
+ const results = await crawl(
+ { url: TEST_SUITE_WEBSITE, limit: 3 },
+ identity,
+ );
+
+ expect(results.warning ?? "").not.toContain("robots.txt");
+ expect((await crawlErrors(results.id, identity)).robotsBlocked).toEqual(
+ [],
+ );
+ },
+ 5 * scrapeTimeout,
+ );
+});
diff --git a/apps/api/src/__tests__/snips/v2/crawl.test.ts b/apps/api/src/__tests__/snips/v2/crawl.test.ts
index 66748bef0..57ad432de 100644
--- a/apps/api/src/__tests__/snips/v2/crawl.test.ts
+++ b/apps/api/src/__tests__/snips/v2/crawl.test.ts
@@ -743,7 +743,7 @@ describe("Crawl tests", () => {
// Check specifically for robots.txt warning
if (results.warning && results.warning.includes("robots.txt")) {
expect(results.warning).toContain("robots.txt");
- expect(results.warning).toContain("/scrape endpoint");
+ expect(results.warning).toContain("robotsBlocked");
}
},
10 * scrapeTimeout,
diff --git a/apps/api/src/__tests__/snips/v2/lib.ts b/apps/api/src/__tests__/snips/v2/lib.ts
index fec7ad9b2..a9ed8a6fc 100644
--- a/apps/api/src/__tests__/snips/v2/lib.ts
+++ b/apps/api/src/__tests__/snips/v2/lib.ts
@@ -503,7 +503,7 @@ export async function asyncCrawlWaitForFinish(
return x.body;
}
-async function crawlErrors(
+export async function crawlErrors(
id: string,
identity: Identity,
): Promise<Exclude<CrawlErrorsResponse, ErrorResponse>> {
diff --git a/apps/api/src/__tests__/snips/v2/scrape-robots.test.ts b/apps/api/src/__tests__/snips/v2/scrape-robots.test.ts
index b5ab41b40..c868b46e0 100644
--- a/apps/api/src/__tests__/snips/v2/scrape-robots.test.ts
+++ b/apps/api/src/__tests__/snips/v2/scrape-robots.test.ts
@@ -48,6 +48,7 @@ describe("Scrape robots policy", () => {
if (result.success) throw new Error("Expected a robots denial");
expect(result.error).toBeInstanceOf(CrawlDenialError);
expect(result.error.message).toBe("URL blocked by robots.txt");
+ expect((result.error as CrawlDenialError).robots).toBe(true);
expect(engines.scrapeURLWithEngine).not.toHaveBeenCalled();
},
scrapeTimeout,
diff --git a/apps/api/src/controllers/v2/__tests__/crawl-status-next.test.ts b/apps/api/src/controllers/v2/__tests__/crawl-status-next.test.ts
index 33b57c168..8edcaa554 100644
--- a/apps/api/src/controllers/v2/__tests__/crawl-status-next.test.ts
+++ b/apps/api/src/controllers/v2/__tests__/crawl-status-next.test.ts
@@ -29,7 +29,7 @@ vi.mock("../../../services/worker/nuq-router", () => ({
}));
vi.mock("../../../services/redis", () => ({
- redisEvictConnection: { smembers: vi.fn().mockResolvedValue([]) },
+ redisEvictConnection: { scard: vi.fn().mockResolvedValue(0) },
}));
vi.mock("../../../lib/request-credits-store", () => ({
diff --git a/apps/api/src/controllers/v2/crawl-status.ts b/apps/api/src/controllers/v2/crawl-status.ts
index ae456e731..88d79eb7f 100644
--- a/apps/api/src/controllers/v2/crawl-status.ts
+++ b/apps/api/src/controllers/v2/crawl-status.ts
@@ -336,15 +336,13 @@ export async function crawlStatusController(
// Check for robots.txt blocked URLs and add warning if found
let warning: string | undefined;
try {
- const robotsBlocked = await redisEvictConnection.smembers(
+ const rbCount = await redisEvictConnection.scard(
"crawl:" + req.params.jobId + ":robots_blocked",
);
- const rbCount = robotsBlocked?.length ?? 0;
// Emit as separate simple logs so no meta is lost in sinks
const statusNow = outputBulkA.status ?? "scraping";
if (rbCount > 0 && statusNow !== "scraping") {
- warning =
- "One or more pages were unable to be crawled because the robots.txt file prevented this. Please use the /scrape endpoint instead.";
+ warning = `One or more pages could not be crawled because the site's robots.txt disallows them. See the robotsBlocked list on GET /v2/${isBatch ? "batch/scrape" : "crawl"}/${req.params.jobId}/errors. Teams with the feature enabled can set ignoreRobotsTxt: true.`;
}
} catch (error) {
// If we can't check robots blocked URLs, continue without warning
diff --git a/apps/api/src/lib/crawl-redis.ts b/apps/api/src/lib/crawl-redis.ts
index d8c4b4c50..9bf1becd1 100644
--- a/apps/api/src/lib/crawl-redis.ts
+++ b/apps/api/src/lib/crawl-redis.ts
@@ -78,8 +78,14 @@ export async function saveCrawl(id: string, crawl: StoredCrawl) {
);
}
-export async function recordRobotsBlocked(crawlId: string, url: string) {
- await redisEvictConnection.sadd("crawl:" + crawlId + ":robots_blocked", url);
+export async function recordRobotsBlocked(crawlId: string, urls: string[]) {
+ if (urls.length === 0) return;
+ for (let i = 0; i < urls.length; i += REDIS_COMMAND_ARG_CHUNK_SIZE) {
+ await redisEvictConnection.sadd(
+ "crawl:" + crawlId + ":robots_blocked",
+ ...urls.slice(i, i + REDIS_COMMAND_ARG_CHUNK_SIZE),
+ );
+ }
await redisEvictConnection.expire(
"crawl:" + crawlId + ":robots_blocked",
24 * 60 * 60,
diff --git a/apps/api/src/lib/error.ts b/apps/api/src/lib/error.ts
index 08e89bdfb..a59e7750a 100644
--- a/apps/api/src/lib/error.ts
+++ b/apps/api/src/lib/error.ts
@@ -357,14 +357,22 @@ export class SitemapError extends TransportableError {
}
export class CrawlDenialError extends TransportableError {
- constructor(public reason: string) {
+ /** True when robots.txt caused the denial. */
+ public robots: boolean;
+
+ constructor(
+ public reason: string,
+ options: { robots?: boolean } = {},
+ ) {
super("CRAWL_DENIAL", reason);
+ this.robots = options.robots ?? false;
}
serialize() {
return {
...super.serialize(),
reason: this.reason,
+ robots: this.robots,
};
}
@@ -372,7 +380,7 @@ export class CrawlDenialError extends TransportableError {
_: ErrorCodes,
data: ReturnType<typeof this.prototype.serialize> & { reason: string },
) {
- const x = new CrawlDenialError(data.reason);
+ const x = new CrawlDenialError(data.reason, { robots: data.robots });
x.stack = data.stack;
return x;
}
diff --git a/apps/api/src/scraper/WebScraper/__tests__/robots-denial.test.ts b/apps/api/src/scraper/WebScraper/__tests__/robots-denial.test.ts
new file mode 100644
index 000000000..c9efe45a7
--- /dev/null
+++ b/apps/api/src/scraper/WebScraper/__tests__/robots-denial.test.ts
@@ -0,0 +1,63 @@
+import { describe, expect, it } from "vitest";
+import { CrawlDenialError } from "../../../lib/error";
+import {
+ deserializeTransportableError,
+ serializeTransportableError,
+} from "../../../lib/error-serde";
+import { isRobotsDenialReason, WebCrawler } from "../crawler";
+
+const BASE = "https://example.com";
+const BLOCKED = BASE + "/blocked/page";
+const EXCLUDED = BASE + "/excluded/page";
+
+function makeCrawler() {
+ const crawler = new WebCrawler({
+ jobId: "robots-denial-test",
+ initialUrl: BASE + "/",
+ excludes: ["^/excluded"],
+ });
+ crawler.importRobotsTxt("User-agent: *\nDisallow: /blocked\nAllow: /\n");
+ return crawler;
+}
+
+describe("robots.txt denials", () => {
+ it("flags only the robots.txt denial in filterLinks", async () => {
+ const { links, denialReasons } = await makeCrawler().filterLinks(
+ [BASE + "/ok", BLOCKED, EXCLUDED],
+ 10,
+ 10,
+ );
+
+ expect(links).toEqual([BASE + "/ok"]);
+ expect(isRobotsDenialReason(denialReasons.get(BLOCKED))).toBe(true);
+ expect(denialReasons.get(EXCLUDED)).toBeDefined();
+ expect(isRobotsDenialReason(denialReasons.get(EXCLUDED))).toBe(false);
+ expect(isRobotsDenialReason(undefined)).toBe(false);
+ });
+
+ it("keeps robots.txt-blocked links during extraction so filterLinks can report them", async () => {
+ const html = `<a href="/ok">ok</a><a href="/blocked/page">blocked</a>`;
+ const extracted = await makeCrawler().extractLinksFromContent(
+ html,
+ BASE + "/",
+ );
+
+ expect(extracted).toContain(BLOCKED);
+ });
+
+ it("keeps the robots marker on CrawlDenialError through serialization", () => {
+ const robots = deserializeTransportableError(
+ serializeTransportableError(
+ new CrawlDenialError("blocked", { robots: true }),
+ ),
+ );
+ const other = deserializeTransportableError(
+ serializeTransportableError(new CrawlDenialError("excluded")),
+ );
+
+ expect(robots).toBeInstanceOf(CrawlDenialError);
+ expect((robots as CrawlDenialError).robots).toBe(true);
+ expect((robots as CrawlDenialError).reason).toBe("blocked");
+ expect((other as CrawlDenialError).robots).toBe(false);
+ });
+});
diff --git a/apps/api/src/scraper/WebScraper/crawler.ts b/apps/api/src/scraper/WebScraper/crawler.ts
index b212e5954..5dbb04296 100644
--- a/apps/api/src/scraper/WebScraper/crawler.ts
+++ b/apps/api/src/scraper/WebScraper/crawler.ts
@@ -51,6 +51,11 @@ enum DenialReason {
NON_WEB_PROTOCOL = "This URL uses a non-web protocol (such as mailto:, tel:, ftp:, ssh:, file:, or telnet:) that Firecrawl cannot scrape. Firecrawl only supports HTTP and HTTPS protocols.",
}
+/** True when a filterLinks denial reason is the robots.txt denial. */
+export function isRobotsDenialReason(reason: string | undefined): boolean {
+ return reason === DenialReason.ROBOTS_TXT;
+}
+
interface FilterLinksResult {
links: string[];
denialReasons: Map<string, string>;
@@ -441,10 +446,7 @@ export class WebCrawler {
if (config.FIRECRAWL_DEBUG_FILTER_LINKS) {
this.logger.debug(`${link} ROBOTS FAIL`);
}
- denialReasons.set(
- link,
- `This URL is blocked by the website's robots.txt file, which instructs crawlers not to access this page. Firecrawl respects robots.txt by default. To crawl this URL anyway, set ignoreRobotsTxt: true in your crawl request (note: this may violate the website's crawling policies).`,
- );
+ denialReasons.set(link, DenialReason.ROBOTS_TXT);
return false;
}
@@ -729,13 +731,17 @@ export class WebCrawler {
return count;
}
- public async filterURL(href: string, url: string): Promise<FilterResult> {
+ public async filterURL(
+ href: string,
+ url: string,
+ skipRobots: boolean = false,
+ ): Promise<FilterResult> {
return await filterUrl({
href: href,
url: url,
baseUrl: this.baseUrl,
excludes: this.excludes,
- ignoreRobotsTxt: this.ignoreRobotsTxt,
+ ignoreRobotsTxt: this.ignoreRobotsTxt || skipRobots,
robotsTxt: this.robotsTxt,
robotsUserAgent: this.robotsUserAgent,
allowExternalContentLinks: this.allowExternalContentLinks,
@@ -747,7 +753,7 @@ export class WebCrawler {
const links = await extractLinks(html);
const filteredLinks: string[] = [];
for (const link of links) {
- const filterResult = await this.filterURL(link, url);
+ const filterResult = await this.filterURL(link, url, true);
if (filterResult.allowed && filterResult.url) {
filteredLinks.push(filterResult.url);
}
@@ -766,7 +772,7 @@ export class WebCrawler {
if (href.match(/^https?:\/[^\/]/)) {
href = href.replace(/^https?:\//, "$&/");
}
- const filterResult = await this.filterURL(href, url);
+ const filterResult = await this.filterURL(href, url, true);
if (filterResult.allowed && filterResult.url) {
links.push(filterResult.url);
}
@@ -793,7 +799,7 @@ export class WebCrawler {
private async extractLinksFromMarkdownContent(text: string, url: string) {
const filteredLinks: string[] = [];
for (const link of extractLinksFromMarkdown(text, url)) {
- const filterResult = await this.filterURL(link, url);
+ const filterResult = await this.filterURL(link, url, true);
if (filterResult.allowed && filterResult.url) {
filteredLinks.push(filterResult.url);
}
diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.ts
index b12b420a6..ad3be357f 100644
--- a/apps/api/src/scraper/scrapeURL/index.ts
+++ b/apps/api/src/scraper/scrapeURL/index.ts
@@ -1433,7 +1433,9 @@ export async function scrapeURL(
setSpanAttributes(span, {
"scrape.blocked_by_robots": true,
});
- throw new CrawlDenialError("URL blocked by robots.txt");
+ throw new CrawlDenialError("URL blocked by robots.txt", {
+ robots: true,
+ });
}
} catch (error) {
if (error instanceof CrawlDenialError) {
diff --git a/apps/api/src/services/worker/scrape-worker.ts b/apps/api/src/services/worker/scrape-worker.ts
index a9aa3a685..58630f933 100644
--- a/apps/api/src/services/worker/scrape-worker.ts
+++ b/apps/api/src/services/worker/scrape-worker.ts
@@ -61,7 +61,10 @@ import { normalizeUrlOnlyHostname } from "../../lib/canonical-url";
import { isUrlBlocked } from "../../scraper/WebScraper/utils/blocklist";
import { generateURLSplits, queryIndexAtSplitLevel } from "../index";
-import { WebCrawler } from "../../scraper/WebScraper/crawler";
+import {
+ isRobotsDenialReason,
+ WebCrawler,
+} from "../../scraper/WebScraper/crawler";
import {
calculateCreditsToBeBilled,
calculateThreatScanCredits,
@@ -645,12 +648,12 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {
linksLength: links.links.length,
});
- // Store robots blocked URLs in Redis set
- for (const [url, reason] of links.denialReasons) {
- if (reason === "URL blocked by robots.txt") {
- await recordRobotsBlocked(job.data.crawl_id, url);
- }
- }
+ await recordRobotsBlocked(
+ job.data.crawl_id,
+ [...links.denialReasons]
+ .filter(([, reason]) => isRobotsDenialReason(reason))
+ .map(([url]) => url),
+ );
// Threat protection: silently skip blocked discovered links
// (cross-domain links included) — the crawl continues. Skipped
@@ -804,10 +807,13 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {
);
if (filterResult.links.length === 0) {
const url = doc.metadata.url ?? doc.metadata.sourceURL!;
+ const denialReason = filterResult.denialReasons.get(url);
const reason =
- filterResult.denialReasons.get(url) ||
+ denialReason ||
`The source URL ("${url}") you provided as the starting point for this crawl is not allowed by your own crawl configuration. This can happen if your includePaths, excludePaths, maxDepth, or other filters exclude the starting URL itself. Please check your crawl configuration to ensure the starting URL is allowed.`;
- throw new CrawlDenialError(reason);
+ throw new CrawlDenialError(reason, {
+ robots: isRobotsDenialReason(denialReason),
+ });
}
}
}
@@ -1050,9 +1056,9 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {
job.data.crawl_id &&
job.data.crawlerOptions !== null &&
error instanceof CrawlDenialError &&
- error.reason === "URL blocked by robots.txt"
+ error.robots
) {
- await recordRobotsBlocked(job.data.crawl_id, job.data.url);
+ await recordRobotsBlocked(job.data.crawl_id, [job.data.url]);
}
} catch (e) {
logger.debug("Failed to record top-level robots block", { e });
diff --git a/apps/test-site/astro.config.mjs b/apps/test-site/astro.config.mjs
index fcf819638..e5af79017 100644
--- a/apps/test-site/astro.config.mjs
+++ b/apps/test-site/astro.config.mjs
@@ -13,5 +13,10 @@ const SITE_URL =
export default defineConfig({
site: SITE_URL,
output: 'static',
- integrations: [mdx(), sitemap()],
+ integrations: [
+ mdx(),
+ // Keep the robots.txt fixture out of the sitemap so that only crawls
+ // that start at /robots-test/ see it.
+ sitemap({ filter: page => !page.includes("/robots-test/") }),
+ ],
});
diff --git a/apps/test-site/src/pages/robots-test/blocked.astro b/apps/test-site/src/pages/robots-test/blocked.astro
new file mode 100644
index 000000000..d402ba000
--- /dev/null
+++ b/apps/test-site/src/pages/robots-test/blocked.astro
@@ -0,0 +1,15 @@
+---
+// Fixture for crawl robots.txt reporting. robots.txt disallows this page.
+---
+
+<!doctype html>
+<html lang="en">
+ <head>
+ <meta charset="utf-8" />
+ <title>Robots Test Blocked</title>
+ </head>
+ <body>
+ <h1>Robots Test Blocked</h1>
+ <p>robots.txt disallows this page.</p>
+ </body>
+</html>
diff --git a/apps/test-site/src/pages/robots-test/index.astro b/apps/test-site/src/pages/robots-test/index.astro
new file mode 100644
index 000000000..7ba10e016
--- /dev/null
+++ b/apps/test-site/src/pages/robots-test/index.astro
@@ -0,0 +1,18 @@
+---
+// Fixture for crawl robots.txt reporting. robots.txt disallows
+// /robots-test/blocked/, and the sitemap leaves out /robots-test/, so only
+// crawls that start here see these pages.
+---
+
+<!doctype html>
+<html lang="en">
+ <head>
+ <meta charset="utf-8" />
+ <title>Robots Test</title>
+ </head>
+ <body>
+ <h1>Robots Test</h1>
+ <p>This page is allowed by robots.txt.</p>
+ <p><a href="/robots-test/blocked/">This link goes to a page that robots.txt disallows.</a></p>
+ </body>
+</html>
diff --git a/apps/test-site/src/pages/robots.txt.ts b/apps/test-site/src/pages/robots.txt.ts
index 74ee15e2d..135e98446 100644
--- a/apps/test-site/src/pages/robots.txt.ts
+++ b/apps/test-site/src/pages/robots.txt.ts
@@ -2,6 +2,7 @@ import type { APIRoute } from "astro";
const getRobotsTxt = (sitemapURL: URL) => `\
User-agent: *
+Disallow: /robots-test/blocked
Allow: /
Sitemap: ${sitemapURL.href}