Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

feat: support custom `userAgent` with `respectRobotsTxtFile` (#3226)

Jindřich Bär354252dee44c
diff --git a/packages/basic-crawler/src/internals/basic-crawler.ts b/packages/basic-crawler/src/internals/basic-crawler.ts
index 203ddc183..2f52f075f 100644
--- a/packages/basic-crawler/src/internals/basic-crawler.ts
+++ b/packages/basic-crawler/src/internals/basic-crawler.ts
@@ -369,8 +369,11 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = BasicCraw
/**
* If set to `true`, the crawler will automatically try to fetch the robots.txt file for each domain,
* and skip those that are not allowed. This also prevents disallowed URLs to be added via `enqueueLinks`.
+ *
+ * If an object is provided, it may contain a `userAgent` property to specify which user-agent
+ * should be used when checking the robots.txt file. If not provided, the default user-agent `*` will be used.
*/
- respectRobotsTxtFile?: boolean;
+ respectRobotsTxtFile?: boolean | { userAgent?: string };
/**
* When a request is skipped for some reason, you can use this callback to act on it.
@@ -556,7 +559,7 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
protected events: EventManager;
protected httpClient: BaseHttpClient;
protected retryOnBlocked: boolean;
- protected respectRobotsTxtFile: boolean;
+ protected respectRobotsTxtFile: boolean | { userAgent?: string };
protected onSkippedRequest?: SkippedRequestCallback;
private _closeEvents?: boolean;
private shouldLogMaxProcessedRequestsExceeded = true;
@@ -594,7 +597,7 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
statusMessageCallback: ow.optional.function,
retryOnBlocked: ow.optional.boolean,
- respectRobotsTxtFile: ow.optional.boolean,
+ respectRobotsTxtFile: ow.optional.any(ow.boolean, ow.object),
onSkippedRequest: ow.optional.function,
httpClient: ow.optional.object,
@@ -1357,7 +1360,9 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
}
const robotsTxtFile = await this.getRobotsTxtFileForUrl(url);
- return !robotsTxtFile || robotsTxtFile.isAllowed(url);
+ const userAgent = typeof this.respectRobotsTxtFile === 'object' ? this.respectRobotsTxtFile?.userAgent : '*';
+
+ return !robotsTxtFile || robotsTxtFile.isAllowed(url, userAgent);
}
protected async getRobotsTxtFileForUrl(url: string): Promise<RobotsTxtFile | undefined> {
diff --git a/test/core/crawlers/basic_crawler.test.ts b/test/core/crawlers/basic_crawler.test.ts
index 4d3a75213..5ab04a286 100644
--- a/test/core/crawlers/basic_crawler.test.ts
+++ b/test/core/crawlers/basic_crawler.test.ts
@@ -25,7 +25,7 @@ import {
} from '@crawlee/basic';
import { RequestState } from '@crawlee/core';
import type { Dictionary } from '@crawlee/utils';
-import { sleep } from '@crawlee/utils';
+import { RobotsTxtFile, sleep } from '@crawlee/utils';
import express from 'express';
import { MemoryStorageEmulator } from 'test/shared/MemoryStorageEmulator';
import type { SetRequired } from 'type-fest';
@@ -1700,6 +1700,65 @@ describe('BasicCrawler', () => {
expect(addRequestsBatchedSpy).toHaveBeenCalledOnce();
});
+ test.each([
+ {
+ testName: 'custom user-agent robots.txt rules',
+ userAgent: 'MyCrawler',
+ visitedUrls: [
+ {
+ url: 'http://example.com/yes',
+ },
+ {
+ url: 'http://example.com/my-crawler/anything',
+ },
+ ],
+ },
+ {
+ testName: 'catch-all robots.txt rules with custom user-agent',
+ userAgent: 'RandomCrawler',
+ visitedUrls: [
+ {
+ url: 'http://example.com/yes',
+ },
+ ],
+ },
+ ])('should respect $testName', async ({ userAgent, visitedUrls }) => {
+ const requestQueue = await RequestQueue.open();
+ const addRequestsBatchedSpy = vitest.spyOn(requestQueue, 'addRequestsBatched');
+
+ const crawler = new (class MockedRobotsTxtCrawler extends BasicCrawler {
+ override async getRobotsTxtFileForUrl(_: string) {
+ return RobotsTxtFile.from(
+ 'http://example.com/robots.txt',
+ `User-agent: *
+ Disallow: /
+ Allow: /yes
+
+ User-agent: MyCrawler
+ Disallow: /no
+ Allow: /my-crawler
+ `,
+ );
+ }
+ })({
+ requestQueue,
+ respectRobotsTxtFile: { userAgent },
+ requestHandler: async () => {},
+ });
+
+ await crawler.addRequests([
+ 'http://example.com/yes', // Allowed by robots.txt
+ 'http://example.com/no', // Blocked by robots.txt for "MyCrawler"
+ 'http://example.com/no-globally', // Blocked by robots.txt, "*" rule
+ 'http://example.com/my-crawler/anything', // Blocked by robots.txt for all user-agents, but allowed for "MyCrawler"
+ ]);
+
+ await expect(localStorageEmulator.getRequestQueueItems()).resolves.toMatchObject(visitedUrls);
+
+ // Should only have added the first request (allowed by robots.txt and within limit)
+ expect(addRequestsBatchedSpy).toHaveBeenCalledOnce();
+ });
+
test('enqueueLinks should respect maxRequestsPerCrawl', async () => {
const requestQueue = await RequestQueue.open();
const addRequestsBatchedSpy = vitest.spyOn(requestQueue, 'addRequestsBatched');