Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(api): check robots.txt before fetching a crawl's start URL (#5005)

Gergő Móricz8d737f12ed39
diff --git a/apps/api/src/__tests__/snips/v2/crawl-robots.test.ts b/apps/api/src/__tests__/snips/v2/crawl-robots.test.ts
index cb1659ede..16232fb07 100644
--- a/apps/api/src/__tests__/snips/v2/crawl-robots.test.ts
+++ b/apps/api/src/__tests__/snips/v2/crawl-robots.test.ts
@@ -1,7 +1,11 @@
+import http from "node:http";
import { describe, expect } from "vitest";
import {
ALLOW_TEST_SUITE_WEBSITE,
concurrentIf,
+ describeIf,
+ HAS_PROXY,
+ TEST_SELF_HOST,
TEST_SUITE_WEBSITE,
} from "../lib";
import { crawl, crawlErrors, Identity, idmux, scrapeTimeout } from "./lib";
@@ -80,3 +84,170 @@ describe("Crawl robots.txt reporting", () => {
5 * scrapeTimeout,
);
});
+
+// A local site whose robots.txt disallows /blocked/, so the tests can count
+// the requests a crawl makes. Proxies and remote engines cannot reach it.
+describeIf(TEST_SELF_HOST && !HAS_PROXY)("Crawl robots.txt requests", () => {
+ const LINKS: Record<string, string[]> = {
+ "/allowed/start/": ["/blocked/from-allowed/"],
+ "/allowed/redirect-links/": ["/redirect/discovered/"],
+ "/blocked/redirect-target/": ["/allowed/behind-redirect/"],
+ };
+ const REDIRECTS: Record<string, string> = {
+ "/redirect/start/": "/blocked/redirect-target/",
+ "/redirect/discovered/": "/blocked/discovered-target/",
+ };
+
+ const hits = new Map<string, number>();
+ const hitsFor = (path: string) => hits.get(path) ?? 0;
+ let server: http.Server;
+ let base: string;
+
+ beforeAll(async () => {
+ server = http.createServer((req, res) => {
+ const path = new URL(req.url ?? "/", "http://fixture").pathname;
+ hits.set(path, hitsFor(path) + 1);
+
+ if (path === "/robots.txt") {
+ res.setHeader("Content-Type", "text/plain");
+ res.end("User-agent: *\nDisallow: /blocked/\n");
+ } else if (path === "/sitemap.xml") {
+ const locs = ["/allowed/from-sitemap/", "/blocked/from-sitemap/"];
+ res.setHeader("Content-Type", "application/xml");
+ res.end(
+ `<?xml version="1.0" encoding="UTF-8"?><urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">${locs.map(loc => `<url><loc>${base}${loc}</loc></url>`).join("")}</urlset>`,
+ );
+ } else if (REDIRECTS[path]) {
+ res.writeHead(302, { Location: REDIRECTS[path] });
+ res.end();
+ } else {
+ const links = (LINKS[path] ?? [])
+ .map(link => `<a href="${link}">${link}</a>`)
+ .join("");
+ res.setHeader("Content-Type", "text/html; charset=utf-8");
+ res.end(
+ `<html><head><title>${path}</title></head><body><h1>${path}</h1>${links}</body></html>`,
+ );
+ }
+ });
+
+ await new Promise<void>(resolve => server.listen(0, "127.0.0.1", resolve));
+ const address = server.address();
+ if (address === null || typeof address === "string") {
+ throw new Error("Failed to start the robots.txt fixture site");
+ }
+ base = `http://127.0.0.1:${address.port}`;
+ });
+
+ afterAll(async () => {
+ server.closeAllConnections();
+ await new Promise(resolve => server.close(resolve));
+ });
+
+ const robotsBlockedPaths = async (crawlId: string) =>
+ (await crawlErrors(crawlId, identity)).robotsBlocked.map(
+ url => new URL(url).pathname,
+ );
+ const crawledPaths = (data: { metadata?: { url?: string } }[]) =>
+ data.map(doc => new URL(doc.metadata?.url ?? base).pathname);
+
+ it.concurrent(
+ "never requests a start URL that robots.txt disallows",
+ async () => {
+ const results = await crawl(
+ { url: base + "/blocked/start/", limit: 5, sitemap: "skip" },
+ identity,
+ false,
+ );
+
+ expect(hitsFor("/blocked/start/")).toBe(0);
+ expect(results.data).toEqual([]);
+ expect(results.warning ?? "").toContain("robotsBlocked");
+ expect(await robotsBlockedPaths(results.id)).toContain("/blocked/start/");
+ },
+ 5 * scrapeTimeout,
+ );
+
+ it.concurrent(
+ "crawls an allowed start URL without requesting the link robots.txt disallows",
+ async () => {
+ const results = await crawl(
+ {
+ url: base + "/allowed/start/",
+ limit: 5,
+ sitemap: "skip",
+ crawlEntireDomain: true,
+ },
+ identity,
+ );
+
+ expect(hitsFor("/allowed/start/")).toBeGreaterThan(0);
+ expect(crawledPaths(results.data)).toEqual(["/allowed/start/"]);
+ expect(hitsFor("/blocked/from-allowed/")).toBe(0);
+ expect(await robotsBlockedPaths(results.id)).toContain(
+ "/blocked/from-allowed/",
+ );
+ },
+ 5 * scrapeTimeout,
+ );
+
+ it.concurrent(
+ "records the disallowed redirect target of a start URL and crawls nothing from it",
+ async () => {
+ const results = await crawl(
+ {
+ url: base + "/redirect/start/",
+ limit: 5,
+ sitemap: "skip",
+ crawlEntireDomain: true,
+ },
+ identity,
+ false,
+ );
+ const blocked = await robotsBlockedPaths(results.id);
+
+ expect(hitsFor("/redirect/start/")).toBeGreaterThan(0);
+ expect(blocked).toContain("/blocked/redirect-target/");
+ expect(blocked).not.toContain("/redirect/start/");
+ expect(results.data).toEqual([]);
+ expect(hitsFor("/allowed/behind-redirect/")).toBe(0);
+ },
+ 5 * scrapeTimeout,
+ );
+
+ it.concurrent(
+ "records the disallowed redirect target of a discovered link",
+ async () => {
+ const results = await crawl(
+ {
+ url: base + "/allowed/redirect-links/",
+ limit: 5,
+ sitemap: "skip",
+ crawlEntireDomain: true,
+ },
+ identity,
+ );
+ const blocked = await robotsBlockedPaths(results.id);
+
+ expect(hitsFor("/redirect/discovered/")).toBeGreaterThan(0);
+ expect(blocked).toContain("/blocked/discovered-target/");
+ expect(blocked).not.toContain("/redirect/discovered/");
+ expect(crawledPaths(results.data)).toEqual(["/allowed/redirect-links/"]);
+ },
+ 5 * scrapeTimeout,
+ );
+
+ it.concurrent(
+ "records a sitemap URL that robots.txt disallows without requesting it",
+ async () => {
+ const results = await crawl({ url: base + "/", limit: 5 }, identity);
+
+ expect(hitsFor("/allowed/from-sitemap/")).toBeGreaterThan(0);
+ expect(hitsFor("/blocked/from-sitemap/")).toBe(0);
+ expect(await robotsBlockedPaths(results.id)).toContain(
+ "/blocked/from-sitemap/",
+ );
+ },
+ 5 * scrapeTimeout,
+ );
+});
diff --git a/apps/api/src/__tests__/snips/v2/scrape-robots.test.ts b/apps/api/src/__tests__/snips/v2/scrape-robots.test.ts
index c868b46e0..0245e20fa 100644
--- a/apps/api/src/__tests__/snips/v2/scrape-robots.test.ts
+++ b/apps/api/src/__tests__/snips/v2/scrape-robots.test.ts
@@ -48,7 +48,9 @@ describe("Scrape robots policy", () => {
if (result.success) throw new Error("Expected a robots denial");
expect(result.error).toBeInstanceOf(CrawlDenialError);
expect(result.error.message).toBe("URL blocked by robots.txt");
- expect((result.error as CrawlDenialError).robots).toBe(true);
+ expect((result.error as CrawlDenialError).robotsBlockedUrl).toBe(
+ TEST_SUITE_WEBSITE,
+ );
expect(engines.scrapeURLWithEngine).not.toHaveBeenCalled();
},
scrapeTimeout,
diff --git a/apps/api/src/lib/error.ts b/apps/api/src/lib/error.ts
index a59e7750a..e8460ddd8 100644
--- a/apps/api/src/lib/error.ts
+++ b/apps/api/src/lib/error.ts
@@ -357,22 +357,22 @@ export class SitemapError extends TransportableError {
}
export class CrawlDenialError extends TransportableError {
- /** True when robots.txt caused the denial. */
- public robots: boolean;
+ /** The URL robots.txt disallows, or null when robots.txt did not cause the denial. */
+ public robotsBlockedUrl: string | null;
constructor(
public reason: string,
- options: { robots?: boolean } = {},
+ options: { robotsBlockedUrl?: string | null } = {},
) {
super("CRAWL_DENIAL", reason);
- this.robots = options.robots ?? false;
+ this.robotsBlockedUrl = options.robotsBlockedUrl ?? null;
}
serialize() {
return {
...super.serialize(),
reason: this.reason,
- robots: this.robots,
+ robotsBlockedUrl: this.robotsBlockedUrl,
};
}
@@ -380,7 +380,9 @@ export class CrawlDenialError extends TransportableError {
_: ErrorCodes,
data: ReturnType<typeof this.prototype.serialize> & { reason: string },
) {
- const x = new CrawlDenialError(data.reason, { robots: data.robots });
+ const x = new CrawlDenialError(data.reason, {
+ robotsBlockedUrl: data.robotsBlockedUrl,
+ });
x.stack = data.stack;
return x;
}
diff --git a/apps/api/src/scraper/WebScraper/__tests__/robots-denial.test.ts b/apps/api/src/scraper/WebScraper/__tests__/robots-denial.test.ts
index c9efe45a7..1ef2c19a1 100644
--- a/apps/api/src/scraper/WebScraper/__tests__/robots-denial.test.ts
+++ b/apps/api/src/scraper/WebScraper/__tests__/robots-denial.test.ts
@@ -4,35 +4,53 @@ import {
deserializeTransportableError,
serializeTransportableError,
} from "../../../lib/error-serde";
-import { isRobotsDenialReason, WebCrawler } from "../crawler";
+import { DenialReason, WebCrawler } from "../crawler";
const BASE = "https://example.com";
const BLOCKED = BASE + "/blocked/page";
const EXCLUDED = BASE + "/excluded/page";
-function makeCrawler() {
+function makeCrawler(ignoreRobotsTxt = false) {
const crawler = new WebCrawler({
jobId: "robots-denial-test",
initialUrl: BASE + "/",
excludes: ["^/excluded"],
+ ignoreRobotsTxt,
});
crawler.importRobotsTxt("User-agent: *\nDisallow: /blocked\nAllow: /\n");
return crawler;
}
describe("robots.txt denials", () => {
- it("flags only the robots.txt denial in filterLinks", async () => {
- const { links, denialReasons } = await makeCrawler().filterLinks(
- [BASE + "/ok", BLOCKED, EXCLUDED],
+ it("lists only the robots.txt denials in filterLinks", async () => {
+ const { links, denialReasons, robotsBlocked } =
+ await makeCrawler().filterLinks(
+ [BASE + "/ok", BLOCKED, EXCLUDED],
+ 10,
+ 10,
+ );
+
+ expect(links).toEqual([BASE + "/ok"]);
+ expect(robotsBlocked).toEqual([BLOCKED]);
+ expect(denialReasons.get(BLOCKED)).toBe(DenialReason.ROBOTS_TXT);
+ expect(denialReasons.get(EXCLUDED)).toBeDefined();
+ });
+
+ it("lists no robots.txt denials when robots.txt is ignored", async () => {
+ const { links, robotsBlocked } = await makeCrawler(true).filterLinks(
+ [BASE + "/ok", BLOCKED],
10,
10,
);
- expect(links).toEqual([BASE + "/ok"]);
- expect(isRobotsDenialReason(denialReasons.get(BLOCKED))).toBe(true);
- expect(denialReasons.get(EXCLUDED)).toBeDefined();
- expect(isRobotsDenialReason(denialReasons.get(EXCLUDED))).toBe(false);
- expect(isRobotsDenialReason(undefined)).toBe(false);
+ expect(links).toEqual([BASE + "/ok", BLOCKED]);
+ expect(robotsBlocked).toEqual([]);
+ });
+
+ it("checks a single URL against robots.txt unless robots.txt is ignored", () => {
+ expect(makeCrawler().isRobotsAllowed(BASE + "/ok")).toBe(true);
+ expect(makeCrawler().isRobotsAllowed(BLOCKED)).toBe(false);
+ expect(makeCrawler(true).isRobotsAllowed(BLOCKED)).toBe(true);
});
it("keeps robots.txt-blocked links during extraction so filterLinks can report them", async () => {
@@ -45,10 +63,10 @@ describe("robots.txt denials", () => {
expect(extracted).toContain(BLOCKED);
});
- it("keeps the robots marker on CrawlDenialError through serialization", () => {
+ it("keeps the robots.txt-blocked URL on CrawlDenialError through serialization", () => {
const robots = deserializeTransportableError(
serializeTransportableError(
- new CrawlDenialError("blocked", { robots: true }),
+ new CrawlDenialError("blocked", { robotsBlockedUrl: BLOCKED }),
),
);
const other = deserializeTransportableError(
@@ -56,8 +74,8 @@ describe("robots.txt denials", () => {
);
expect(robots).toBeInstanceOf(CrawlDenialError);
- expect((robots as CrawlDenialError).robots).toBe(true);
+ expect((robots as CrawlDenialError).robotsBlockedUrl).toBe(BLOCKED);
expect((robots as CrawlDenialError).reason).toBe("blocked");
- expect((other as CrawlDenialError).robots).toBe(false);
+ expect((other as CrawlDenialError).robotsBlockedUrl).toBeNull();
});
});
diff --git a/apps/api/src/scraper/WebScraper/crawler.ts b/apps/api/src/scraper/WebScraper/crawler.ts
index 5dbb04296..78de01bd7 100644
--- a/apps/api/src/scraper/WebScraper/crawler.ts
+++ b/apps/api/src/scraper/WebScraper/crawler.ts
@@ -37,7 +37,8 @@ interface FilterResult {
denialReason?: string;
}
-enum DenialReason {
+/** @public filterLinks also reads the members by key, which knip cannot see. */
+export enum DenialReason {
DEPTH_LIMIT = "This URL exceeds the maximum crawl depth you configured. The URL's depth (number of path segments) is greater than the maxDepth parameter. To crawl this URL, increase the maxDepth value in your crawl request.",
EXCLUDE_PATTERN = "This URL's path matches one of the regex patterns you provided in the excludePaths parameter. URLs matching excludePaths are intentionally skipped during crawling. If this URL should be crawled, adjust your excludePaths patterns.",
INCLUDE_PATTERN = "This URL's path does not match any of the regex patterns you provided in the includePaths parameter. When includePaths is specified, only URLs matching at least one pattern are crawled. If this URL should be crawled, add a matching pattern to includePaths or remove the includePaths restriction.",
@@ -51,14 +52,11 @@ enum DenialReason {
NON_WEB_PROTOCOL = "This URL uses a non-web protocol (such as mailto:, tel:, ftp:, ssh:, file:, or telnet:) that Firecrawl cannot scrape. Firecrawl only supports HTTP and HTTPS protocols.",
}
-/** True when a filterLinks denial reason is the robots.txt denial. */
-export function isRobotsDenialReason(reason: string | undefined): boolean {
- return reason === DenialReason.ROBOTS_TXT;
-}
-
interface FilterLinksResult {
links: string[];
denialReasons: Map<string, string>;
+ /** The links robots.txt disallows. */
+ robotsBlocked: string[];
}
export class WebCrawler {
@@ -199,12 +197,16 @@ export class WebCrawler {
`This URL was not crawled because the maximum discovery depth (${this.maxDiscoveryDepth}) has been reached. Discovery depth counts how many 'hops' from the starting URL a page is. To crawl more pages, increase the maxDiscoveryDepth value in your crawl request.`,
);
});
- return { links: [], denialReasons };
+ return { links: [], denialReasons, robotsBlocked: [] };
}
// If the initial URL is a sitemap.xml, skip filtering
if (this.initialUrl.endsWith("sitemap.xml") && fromMap) {
- return { links: sitemapLinks.slice(0, limit), denialReasons };
+ return {
+ links: sitemapLinks.slice(0, limit),
+ denialReasons,
+ robotsBlocked: [],
+ };
}
try {
@@ -301,6 +303,9 @@ export class WebCrawler {
return {
links: res.links,
denialReasons: fancyDenialReasons,
+ robotsBlocked: Object.keys(res.denialReasons).filter(
+ link => res.denialReasons[link] === "ROBOTS_TXT",
+ ),
};
} catch (error) {
this.logger.error("Error filtering links in Rust, falling back to JS", {
@@ -433,12 +438,8 @@ export class WebCrawler {
}
}
- const isAllowed = this.isRobotsAllowed(
- link,
- this.ignoreRobotsTxt || skipRobots,
- );
// Check if the link is disallowed by robots.txt
- if (!isAllowed) {
+ if (!skipRobots && !this.isRobotsAllowed(link)) {
this.logger.debug(`Link disallowed by robots.txt: ${link}`, {
method: "filterLinks",
link,
@@ -469,7 +470,13 @@ export class WebCrawler {
})
.slice(0, limit);
- return { links: filteredLinks, denialReasons };
+ return {
+ links: filteredLinks,
+ denialReasons,
+ robotsBlocked: [...denialReasons.keys()].filter(
+ link => denialReasons.get(link) === DenialReason.ROBOTS_TXT,
+ ),
+ };
}
public async getRobotsTxt(
@@ -844,17 +851,16 @@ export class WebCrawler {
return await this.extractLinksFromHTMLCheerio(html, url);
}
- private isRobotsAllowed(
- url: string,
- ignoreRobotsTxt: boolean = false,
- ): boolean {
- return ignoreRobotsTxt
- ? true
- : isUrlAllowedByRobots(
- url,
- this.robots,
- this.robotsUserAgent ? [this.robotsUserAgent] : undefined,
- );
+ /** Checks the crawl's robots.txt, honoring ignoreRobotsTxt. */
+ public isRobotsAllowed(url: string): boolean {
+ return (
+ this.ignoreRobotsTxt ||
+ isUrlAllowedByRobots(
+ url,
+ this.robots,
+ this.robotsUserAgent ? [this.robotsUserAgent] : undefined,
+ )
+ );
}
public isFile(url: string): boolean {
diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.ts
index ad3be357f..c5d7bda25 100644
--- a/apps/api/src/scraper/scrapeURL/index.ts
+++ b/apps/api/src/scraper/scrapeURL/index.ts
@@ -1434,7 +1434,7 @@ export async function scrapeURL(
"scrape.blocked_by_robots": true,
});
throw new CrawlDenialError("URL blocked by robots.txt", {
- robots: true,
+ robotsBlockedUrl: urlToCheck,
});
}
} catch (error) {
diff --git a/apps/api/src/services/worker/scrape-worker.ts b/apps/api/src/services/worker/scrape-worker.ts
index 58630f933..bfb7d562c 100644
--- a/apps/api/src/services/worker/scrape-worker.ts
+++ b/apps/api/src/services/worker/scrape-worker.ts
@@ -61,10 +61,7 @@ import { normalizeUrlOnlyHostname } from "../../lib/canonical-url";
import { isUrlBlocked } from "../../scraper/WebScraper/utils/blocklist";
import { generateURLSplits, queryIndexAtSplitLevel } from "../index";
-import {
- isRobotsDenialReason,
- WebCrawler,
-} from "../../scraper/WebScraper/crawler";
+import { DenialReason, WebCrawler } from "../../scraper/WebScraper/crawler";
import {
calculateCreditsToBeBilled,
calculateThreatScanCredits,
@@ -437,6 +434,21 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {
if (sc && sc.cancelled) {
throw new JobCancelledError();
}
+
+ // Discovered, sitemap and index URLs pass filterLinks before they are
+ // queued. The start URL is not, so check it here, before it is fetched.
+ if (
+ job.data.isCrawlSourceScrape &&
+ !crawlToCrawler(
+ job.data.crawl_id,
+ sc,
+ (await getACUCTeam(job.data.team_id))?.flags ?? null,
+ ).isRobotsAllowed(job.data.url)
+ ) {
+ throw new CrawlDenialError(DenialReason.ROBOTS_TXT, {
+ robotsBlockedUrl: job.data.url,
+ });
+ }
}
let timeoutHandle: NodeJS.Timeout | null = null;
@@ -578,6 +590,14 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {
normalizeURL(doc.metadata.sourceURL, sc) &&
crawler // only on crawls, don't care on batch scrape
) {
+ // The engine has already followed the redirect, so this can only
+ // keep a disallowed target out of the crawl.
+ if (!crawler.isRobotsAllowed(doc.metadata.url)) {
+ throw new CrawlDenialError(DenialReason.ROBOTS_TXT, {
+ robotsBlockedUrl: doc.metadata.url,
+ });
+ }
+
const filterResult = await crawler!.filterURL(
doc.metadata.url,
doc.metadata.sourceURL,
@@ -648,12 +668,7 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {
linksLength: links.links.length,
});
- await recordRobotsBlocked(
- job.data.crawl_id,
- [...links.denialReasons]
- .filter(([, reason]) => isRobotsDenialReason(reason))
- .map(([url]) => url),
- );
+ await recordRobotsBlocked(job.data.crawl_id, links.robotsBlocked);
// Threat protection: silently skip blocked discovered links
// (cross-domain links included) — the crawl continues. Skipped
@@ -807,12 +822,13 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {
);
if (filterResult.links.length === 0) {
const url = doc.metadata.url ?? doc.metadata.sourceURL!;
- const denialReason = filterResult.denialReasons.get(url);
const reason =
- denialReason ||
+ filterResult.denialReasons.get(url) ||
`The source URL ("${url}") you provided as the starting point for this crawl is not allowed by your own crawl configuration. This can happen if your includePaths, excludePaths, maxDepth, or other filters exclude the starting URL itself. Please check your crawl configuration to ensure the starting URL is allowed.`;
throw new CrawlDenialError(reason, {
- robots: isRobotsDenialReason(denialReason),
+ robotsBlockedUrl: filterResult.robotsBlocked.includes(url)
+ ? url
+ : null,
});
}
}
@@ -1056,9 +1072,9 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {
job.data.crawl_id &&
job.data.crawlerOptions !== null &&
error instanceof CrawlDenialError &&
- error.robots
+ error.robotsBlockedUrl !== null
) {
- await recordRobotsBlocked(job.data.crawl_id, [job.data.url]);
+ await recordRobotsBlocked(job.data.crawl_id, [error.robotsBlockedUrl]);
}
} catch (e) {
logger.debug("Failed to record top-level robots block", { e });
@@ -1258,6 +1274,7 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {
}
async function kickoffGetIndexLinks(
+ crawlId: string,
sc: StoredCrawl,
crawler: WebCrawler,
url: string,
@@ -1276,15 +1293,15 @@ async function kickoffGetIndexLinks(
sc.crawlerOptions.limit ?? 10000,
);
- const validIndexLinksResult = await crawler.filterLinks(
+ const { links, robotsBlocked } = await crawler.filterLinks(
index,
sc.crawlerOptions.limit ?? 10000,
sc.crawlerOptions.maxDepth ?? 10,
false,
);
- const validIndexLinks = validIndexLinksResult.links;
+ await recordRobotsBlocked(crawlId, robotsBlocked);
- return validIndexLinks;
+ return links;
}
async function addKickoffSitemapJob(
@@ -1454,7 +1471,12 @@ async function processKickoffJob(job: NuQJob<ScrapeJobKickoff>) {
}
}
- let indexLinks = await kickoffGetIndexLinks(sc, crawler, job.data.url);
+ let indexLinks = await kickoffGetIndexLinks(
+ job.data.crawl_id,
+ sc,
+ crawler,
+ job.data.url,
+ );
// Threat protection: skip blocked index-sourced discoveries (URL-level
// checks; first-time blocks are recorded and billed, see below).
@@ -1619,14 +1641,14 @@ async function processKickoffSitemapJob(job: NuQJob<ScrapeJobKickoffSitemap>) {
isPreCrawl: sc.internalOptions?.isPreCrawl ?? false,
});
- let passingURLs = (
- await crawler.filterLinks(
- results.urls.map(x => x.href),
- Infinity,
- sc.crawlerOptions.maxDepth ?? 10,
- false,
- )
- ).links;
+ const sitemapLinks = await crawler.filterLinks(
+ results.urls.map(x => x.href),
+ Infinity,
+ sc.crawlerOptions.maxDepth ?? 10,
+ false,
+ );
+ await recordRobotsBlocked(job.data.crawl_id, sitemapLinks.robotsBlocked);
+ let passingURLs = sitemapLinks.links;
// Threat protection: skip blocked sitemap entries — the crawl continues;
// skipped URLs + decisions are recorded as crawl bookkeeping (which also