Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

feat(core): add `userAgent` parameter to `RobotsFile.isAllowed()` + `RobotsFile.from()` helper (#2338)

Martin Adámek343c159f2054
diff --git a/packages/utils/src/internals/robots.ts b/packages/utils/src/internals/robots.ts
index ac7ec6189..30a69e519 100644
--- a/packages/utils/src/internals/robots.ts
+++ b/packages/utils/src/internals/robots.ts
@@ -9,7 +9,7 @@ import { Sitemap } from './sitemap';
let HTTPError: typeof HTTPErrorClass;
/**
- * Loads and queries information from a robots.txt file.
+ * Loads and queries information from a [robots.txt file](https://en.wikipedia.org/wiki/Robots.txt).
*
* **Example usage:**
* ```javascript
@@ -35,7 +35,7 @@ export class RobotsFile {
/**
* Determine the location of a robots.txt file for a URL and fetch it.
* @param url the URL to fetch robots.txt for
- * @param proxyUrl a proxy to be used for fetching the robots.txt file
+ * @param [proxyUrl] a proxy to be used for fetching the robots.txt file
*/
static async find(url: string, proxyUrl?: string): Promise<RobotsFile> {
const robotsFileUrl = new URL(url);
@@ -45,6 +45,16 @@ export class RobotsFile {
return RobotsFile.load(robotsFileUrl.toString(), proxyUrl);
}
+ /**
+ * Allows providing the URL and robotx.txt content explicitly instead of loading it from the target site.
+ * @param url the URL for robots.txt file
+ * @param content contents of robots.txt
+ * @param [proxyUrl] a proxy to be used for fetching the robots.txt file
+ */
+ static from(url: string, content: string, proxyUrl?: string): RobotsFile {
+ return new RobotsFile(robotsParser(url, content), proxyUrl);
+ }
+
protected static async load(url: string, proxyUrl?: string): Promise<RobotsFile> {
if (!HTTPError) {
HTTPError = (await import('got-scraping')).HTTPError;
@@ -70,9 +80,10 @@ export class RobotsFile {
/**
* Check if a URL should be crawled by robots.
* @param url the URL to check against the rules in robots.txt
+ * @param [userAgent] relevant user agent, default to `*`
*/
- isAllowed(url: string): boolean {
- return this.robots.isAllowed(url, '*') ?? false;
+ isAllowed(url: string, userAgent = '*'): boolean {
+ return this.robots.isAllowed(url, userAgent) ?? false;
}
/**
diff --git a/packages/utils/test/robots.test.ts b/packages/utils/test/robots.test.ts
index b231c0da8..3faee2e90 100644
--- a/packages/utils/test/robots.test.ts
+++ b/packages/utils/test/robots.test.ts
@@ -48,4 +48,17 @@ describe('RobotsFile', () => {
const robots = await RobotsFile.find('http://not-exists.com/robots.txt');
expect(robots.getSitemaps()).toEqual(['http://not-exists.com/sitemap_1.xml', 'http://not-exists.com/sitemap_2.xml']);
});
+
+ it('parses allow/deny directives from explicitly provided robots.txt contents', async () => {
+ const contents = `User-agent: *',
+Disallow: *deny_all/
+crawl-delay: 10
+User-agent: Googlebot
+Disallow: *deny_googlebot/`;
+ const robots = RobotsFile.from('http://not-exists.com/robots.txt', contents);
+ expect(robots.isAllowed('http://not-exists.com/something/page.html')).toBe(true);
+ expect(robots.isAllowed('http://not-exists.com/deny_googlebot/page.html')).toBe(true);
+ expect(robots.isAllowed('http://not-exists.com/deny_googlebot/page.html', 'Googlebot')).toBe(false);
+ expect(robots.isAllowed('http://not-exists.com/deny_all/page.html')).toBe(true);
+ });
});