Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Created crawler use proposal in CheerioCrawler class. (#661)
petrpatek804d8ccd6555
diff --git a/src/crawlers/cheerio_crawler.js b/src/crawlers/cheerio_crawler.jsindex e3e022bdd..904e6bd7c 100644--- a/src/crawlers/cheerio_crawler.js+++ b/src/crawlers/cheerio_crawler.js@@ -14,6 +14,7 @@ import { addTimeoutToPromise, parseContentTypeFromResponse } from '../utils';import * as utilsRequest from '../utils_request'; // eslint-disable-line import/no-duplicatesimport BasicCrawler from './basic_crawler'; // eslint-disable-line import/no-duplicatesimport defaultLog from '../utils_log';+import CrawlerExtension from './crawler_extension';// TYPE IMPORTS/* eslint-disable no-unused-vars,import/named,import/no-duplicates,import/order */@@ -396,6 +397,7 @@ class CheerioCrawler {this.prepareRequestFunction = prepareRequestFunction;this.persistCookiesPerSession = persistCookiesPerSession;this.useSessionPool = useSessionPool;+ this.sessionPoolOptions = sessionPoolOptions;/** @ignore */this.basicCrawler = new BasicCrawler({@@ -438,6 +440,42 @@ class CheerioCrawler {await this.isRunningPromise;}+ /**+ * Function for attaching CrawlerExtensions such as the Unblockers.+ * @param extension - Crawler extension that overrides the crawler configuration.+ */+ use(extension) {+ const inheritsFromCrawlerExtension = extension instanceof CrawlerExtension;++ if (!inheritsFromCrawlerExtension) {+ throw new Error('Object passed to the "use" method does not inherit from the "CrawlerExtension" abstract class.');+ }++ const extensionOptions = extension.getCrawlerOptions();++ for (const [key, value] of Object.entries(extensionOptions)) {+ const isConfigurable = this.hasOwnProperty(key); // eslint-disable-line+ const originalType = typeof this[key];+ const extensionType = typeof value; // What if we want to null something? It is really needed?+ const isSameType = originalType === extensionType || value == null; // fast track for deleting keys+ const exists = this[key] != null;++ if (!isConfigurable) { // Test if the property can be configured on the crawler+ throw new Error(`${extension.name} tries to set property "${key}" that is not configurable on CheerioCrawler instance.`);+ }++ if (!isSameType && exists) { // Assuming that extensions will only add up configuration+ throw new Error(+ `${extension.name} tries to set property of different type "${extensionType}". "CheerioCrawler.${key}: ${originalType}".`,+ );+ }++ this.log.warning(`${extension.name} is overriding "CheerioCrawler.${key}: ${originalType}" with ${value}.`);++ this[key] = value;+ }+ }+/*** Wrapper around handlePageFunction that opens and closes pages etc.*diff --git a/src/crawlers/crawler_extension.js b/src/crawlers/crawler_extension.jsnew file mode 100644index 000000000..5ac5602c9--- /dev/null+++ b/src/crawlers/crawler_extension.js@@ -0,0 +1,15 @@+import defaultLog from '../utils_log';++/**+ * Abstract class with pre-defined method to connect to the Crawlers class by the "use" crawler method.+ */+export default class CrawlerExtension {+ constructor() {+ this.name = this.constructor.name;+ this.log = defaultLog.child({ prefix: this.name });+ }++ getCrawlerOptions() {+ throw new Error(`${this.name} has not implemented "getCrawlerOptions" method.`);+ }+}diff --git a/test/crawlers/cheerio_crawler.test.js b/test/crawlers/cheerio_crawler.test.jsindex 2813956a1..277784c59 100644--- a/test/crawlers/cheerio_crawler.test.js+++ b/test/crawlers/cheerio_crawler.test.js@@ -13,6 +13,7 @@ import { Session } from '../../build/session_pool/session';import { STATUS_CODES_BLOCKED } from '../../build/constants';import LocalStorageDirEmulator from '../local_storage_dir_emulator';import * as utilsRequest from '../../build/utils_request';+import CrawlerExtension from '../../build/crawlers/crawler_extension';// Add common props to mocked request responses.const responseMock = {@@ -895,6 +896,81 @@ describe('CheerioCrawler', () => {stub.restore();});});++ describe('use', () => {+ const sources = ['http://example.com/'];+ let requestList;++ class DummyExtension extends CrawlerExtension {+ constructor(options) {+ super();+ this.options = options;+ }++ getCrawlerOptions() {+ return this.options;+ }+ }++ beforeEach(async () => {+ requestList = await Apify.openRequestList(null, sources.slice());+ });++ test('should throw if "CrawlerExtension" class is not used', () => {+ const cheerioCrawler = new Apify.CheerioCrawler({+ requestList,+ maxRequestRetries: 0,+ handlePageFunction: async () => {+ },+ handleFailedRequestFunction: async () => {+ },+ });+ expect(+ () => cheerioCrawler.use({}),+ ).toThrow('Object passed to the "use" method does not inherit from the "CrawlerExtension" abstract class.');+ });++ test('Should throw if "CrawlerExtension" is trying to override non existing property', () => {+ const extension = new DummyExtension({+ doesNotExist: true,+ });+ const cheerioCrawler = new Apify.CheerioCrawler({+ requestList,+ maxRequestRetries: 0,+ handlePageFunction: async () => {},+ handleFailedRequestFunction: async () => {},+ });+ expect(+ () => cheerioCrawler.use(extension),+ )+ .toThrow('DummyExtension tries to set property "doesNotExist" that is not configurable on CheerioCrawler instance.');+ });++ test('should override crawler properties', () => {+ const prepareRequestFunction = async () => ({});+ const extension = new DummyExtension({+ useSessionPool: true,+ prepareRequestFunction,+ apifyProxyGroups: ['SHADER'],+ handlePageFunction: undefined,+ });+ const cheerioCrawler = new Apify.CheerioCrawler({+ requestList,+ useSessionPool: false,+ maxRequestRetries: 0,+ handlePageFunction: async () => {+ },+ handleFailedRequestFunction: async () => {+ },+ });+ expect(cheerioCrawler.useSessionPool).toEqual(false);+ cheerioCrawler.use(extension);+ expect(cheerioCrawler.useSessionPool).toEqual(true);+ expect(cheerioCrawler.prepareRequestFunction).toEqual(prepareRequestFunction);+ expect(cheerioCrawler.handlePageFunction).toBeUndefined();+ expect(cheerioCrawler.apifyProxyGroups[0]).toEqual('SHADER');+ });+ });});async function getRequestListForMock(port, mockData, pathName = 'mock') {diff --git a/test/crawlers/crawler_extension.test.js b/test/crawlers/crawler_extension.test.jsnew file mode 100644index 000000000..66670b7f8--- /dev/null+++ b/test/crawlers/crawler_extension.test.js@@ -0,0 +1,17 @@+import CrawlerExtension from '../../build/crawlers/crawler_extension';++describe('CrawlerExtension', () => {+ test('should work', () => {+ class MyExtension extends CrawlerExtension {+ constructor(options) {+ super();+ this.options = options;+ }+ }+ const myExtension = new MyExtension();+ expect(myExtension.name).toEqual('MyExtension');+ expect(() => myExtension.getCrawlerOptions()).toThrow(`${myExtension.name} has not implemented "getCrawlerOptions" method.`);+ expect(myExtension.log.info).toBeDefined();+ expect(myExtension.log.options.prefix).toEqual('MyExtension');+ });+});