Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
import { FetchHttpClient } from '@crawlee/http-client';
import nock from 'nock';
import { afterEach, beforeEach, describe, expect, it } from 'vitest';
import { RobotsTxtFile } from '../../packages/utils/src/internals/robots.js';
const httpClient = new FetchHttpClient();
// Never replies within the lifetime of a test, so only the timeout/abort can settle the promise.
const neverReplies = 'http://never-replies.com';
describe('RobotsTxtFile', () => {
beforeEach(() => {
nock.disableNetConnect();
nock('http://not-exists.com')
.persist()
.get('/robots.txt')
.delay(500)
.reply(
200,
[
'User-agent: *',
'Disallow: *deny_all/',
'crawl-delay: 10',
'User-agent: Googlebot',
'Disallow: *deny_googlebot/',
'crawl-delay: 1',
'user-agent: Mozilla',
'crawl-delay: 2',
'sitemap: http://not-exists.com/sitemap_1.xml',
'sitemap: http://not-exists.com/sitemap_2.xml',
].join('\n'),
)
.get('*')
.reply(404);
nock(neverReplies).persist().get('/robots.txt').delay(30_000).reply(200, 'User-agent: *');
});
afterEach(() => {
nock.abortPendingRequests();
nock.cleanAll();
nock.enableNetConnect();
});
it('generates the correct robots.txt URL', async () => {
const robots = await RobotsTxtFile.find('http://not-exists.com/nested/index.html', { httpClient });
expect(robots.getSitemaps()).not.toHaveLength(0);
});
it('parses allow/deny directives from robots.txt', async () => {
const robots = await RobotsTxtFile.find('http://not-exists.com/robots.txt', { httpClient });
console.log(robots.isAllowed('https://crawlee.dev'));
expect(robots.isAllowed('http://not-exists.com/something/page.html')).toBe(true);
expect(robots.isAllowed('http://not-exists.com/deny_googlebot/page.html')).toBe(true);
expect(robots.isAllowed('http://not-exists.com/deny_all/page.html')).toBe(false);
});
it('extracts sitemap urls', async () => {
const robots = await RobotsTxtFile.find('http://not-exists.com/robots.txt', { httpClient });
expect(robots.getSitemaps()).toEqual([
'http://not-exists.com/sitemap_1.xml',
'http://not-exists.com/sitemap_2.xml',
]);
});
it.each([401, 403, 404, 410])('treats an unavailable robots.txt (HTTP %i) as allow-all', async (status) => {
nock('http://no-robots.com').get('/robots.txt').reply(status, 'User-agent: *\nDisallow: /');
const robots = await RobotsTxtFile.find('http://no-robots.com/some/page.html', { httpClient });
expect(robots.isAllowed('http://no-robots.com/some/page.html')).toBe(true);
expect(robots.getSitemaps()).toEqual([]);
expect(robots.getCrawlDelay()).toBeUndefined();
});
it.each([500, 503])('treats an unreachable robots.txt (HTTP %i) as disallow-all', async (status) => {
nock('http://broken-robots.com').get('/robots.txt').reply(status, 'User-agent: *\nAllow: /');
const robots = await RobotsTxtFile.find('http://broken-robots.com/some/page.html', { httpClient });
expect(robots.isAllowed('http://broken-robots.com/some/page.html')).toBe(false);
expect(robots.getSitemaps()).toEqual([]);
expect(robots.getCrawlDelay()).toBeUndefined();
});
it('still rejects on other non-2xx responses', async () => {
nock('http://broken-robots.com').get('/robots.txt').reply(304);
await expect(RobotsTxtFile.find('http://broken-robots.com/', { httpClient })).rejects.toThrow(/HTTP 304/);
});
it('respects user-set timeout', async () => {
const start = Date.now();
const robots = RobotsTxtFile.find(`${neverReplies}/robots.txt`, { timeoutMillis: 200 });
await expect(robots).rejects.toThrow(/timeout/i);
const elapsed = Date.now() - start;
expect(elapsed).toBeGreaterThanOrEqual(200);
expect(elapsed).toBeLessThan(10_000);
});
it('respects AbortSignal parameter', async () => {
const controller = new AbortController();
setTimeout(() => controller.abort(), 200);
const start = Date.now();
const robots = RobotsTxtFile.find(`${neverReplies}/robots.txt`, { signal: controller.signal });
await expect(robots).rejects.toThrow(/aborted/i);
const elapsed = Date.now() - start;
expect(elapsed).toBeGreaterThanOrEqual(200);
expect(elapsed).toBeLessThan(10_000);
});
it('respects AbortSignal parameter and timeout together', async () => {
const controller = new AbortController();
const start = Date.now();
const robots = RobotsTxtFile.find(`${neverReplies}/robots.txt`, {
signal: controller.signal,
timeoutMillis: 200,
});
await expect(robots).rejects.toThrow(/timeout/i);
const elapsed = Date.now() - start;
expect(elapsed).toBeGreaterThanOrEqual(200);
expect(elapsed).toBeLessThan(10_000);
});
it('drops cross-host and non-http(s) sitemap directives under the default same-hostname strategy', () => {
const content = [
'User-agent: *',
'Sitemap: http://not-exists.com/legit.xml',
'Sitemap: http://other.test/cross.xml',
'Sitemap: mailto:[email protected]',
'Sitemap: ftp://not-exists.com/cross.xml',
].join('\n');
const robots = RobotsTxtFile.from('http://not-exists.com/robots.txt', content);
expect(robots.getSitemaps()).toEqual(['http://not-exists.com/legit.xml']);
});
it('keeps cross-host http sitemaps but still drops non-http(s) ones with enqueueStrategy "all"', () => {
const content = [
'User-agent: *',
'Sitemap: http://not-exists.com/legit.xml',
'Sitemap: http://other.test/cross.xml',
'Sitemap: mailto:[email protected]',
'Sitemap: ftp://not-exists.com/cross.xml',
].join('\n');
const robots = RobotsTxtFile.from('http://not-exists.com/robots.txt', content);
expect(new Set(robots.getSitemaps({ enqueueStrategy: 'all' }))).toEqual(
new Set(['http://not-exists.com/legit.xml', 'http://other.test/cross.xml']),
);
});
it('getSitemaps does not throw when the robots.txt url is not absolute', () => {
// A non-absolute url must not crash getSitemaps().
const robots = RobotsTxtFile.from('not-exists.com/robots.txt', 'Sitemap: http://not-exists.com/legit.xml');
expect(() => robots.getSitemaps()).not.toThrow();
});
it('parses allow/deny directives from explicitly provided robots.txt contents', async () => {
const contents = `User-agent: *',
Disallow: *deny_all/
crawl-delay: 10
User-agent: Googlebot
Disallow: *deny_googlebot/`;
const robots = RobotsTxtFile.from('http://not-exists.com/robots.txt', contents);
expect(robots.isAllowed('http://not-exists.com/something/page.html')).toBe(true);
expect(robots.isAllowed('http://not-exists.com/deny_googlebot/page.html')).toBe(true);
expect(robots.isAllowed('http://not-exists.com/deny_googlebot/page.html', 'Googlebot')).toBe(false);
expect(robots.isAllowed('http://not-exists.com/deny_all/page.html')).toBe(true);
});
});