Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Feature/crawling context (#813)
Jan Bártade8b708366b1
diff --git a/src/crawlers/basic_crawler.js b/src/crawlers/basic_crawler.jsindex 1d4e46143..1fcb098cd 100644--- a/src/crawlers/basic_crawler.js+++ b/src/crawlers/basic_crawler.js@@ -425,9 +425,13 @@ class BasicCrawler {const statisticsId = request.id || request.uniqueKey;this.stats.startJob(statisticsId);++ // Shared crawling context+ const crawlingContext = { request, autoscaledPool: this.autoscaledPool, session };+try {await addTimeoutToPromise(- this.handleRequestFunction({ request, autoscaledPool: this.autoscaledPool, session }),+ this.handleRequestFunction(crawlingContext),this.handleRequestTimeoutMillis,`handleRequestFunction timed out after ${this.handleRequestTimeoutMillis / 1000} seconds.`,);@@ -439,7 +443,7 @@ class BasicCrawler {if (session) session.markGood();} catch (err) {try {- await this._requestFunctionErrorHandler(err, request, source);+ await this._requestFunctionErrorHandler(err, crawlingContext, source);} catch (secondaryError) {this.log.exception(secondaryError, 'runTaskFunction error handler threw an exception. '+ 'This places the crawler and its underlying storages into an unknown state and crawling will be terminated. '@@ -485,12 +489,13 @@ class BasicCrawler {/*** Handles errors thrown by user provided handleRequestFunction()* @param {Error} error- * @param {Request} request+ * @param {object} crawlingContext* @param {(RequestList|RequestQueue)} source* @return {Promise<boolean|void|QueueOperationInfo>} willBeRetried* @ignore*/- async _requestFunctionErrorHandler(error, request, source) {+ async _requestFunctionErrorHandler(error, crawlingContext, source) {+ const { request } = crawlingContext;request.pushErrorMessage(error);// Reclaim and retry request if flagged as retriable and retryCount is not exceeded.@@ -510,7 +515,8 @@ class BasicCrawler {this.handledRequestsCount++;await source.markRequestHandled(request);this.stats.failJob(request.id || request.url);- return this.handleFailedRequestFunction({ request, error }); // This function prints an error message.+ crawlingContext.error = error;+ return this.handleFailedRequestFunction(crawlingContext); // This function prints an error message.}/**@@ -560,6 +566,9 @@ export default BasicCrawler;/*** @typedef HandleFailedRequestInput- * @property {Request} request The original {Request} object.* @property {Error} error The Error thrown by `handleRequestFunction`.+ * @property {Request} request The original {Request} object.+ * @property {AutoscaledPool} autoscaledPool+ * @property {Session} [session]+ * @property {ProxyInfo} [proxyInfo]*/diff --git a/src/crawlers/cheerio_crawler.js b/src/crawlers/cheerio_crawler.jsindex 7bf97646d..4b19be866 100644--- a/src/crawlers/cheerio_crawler.js+++ b/src/crawlers/cheerio_crawler.js@@ -26,7 +26,6 @@ import { ProxyConfiguration, ProxyInfo } from '../proxy_configuration';import { RequestQueue } from '../storages/request_queue';import { Session } from '../session_pool/session';import { SessionPoolOptions } from '../session_pool/session_pool';-import { RequestAsBrowserOptions } from '../utils_request';import { validators } from '../validators';/* eslint-enable no-unused-vars,import/named,import/no-duplicates,import/order */@@ -119,7 +118,8 @@ const DEFAULT_AUTOSCALED_POOL_OPTIONS = {* ```* {* request: Request,- * session: Session+ * autoscaledPool: AutoscaledPool,+ * session: Session,* }* ```* where the {@link Request} instance corresponds to the initialized request@@ -146,8 +146,17 @@ const DEFAULT_AUTOSCALED_POOL_OPTIONS = {* The function receives the following object as an argument:* ```* {- * request: Request,* error: Error,+ * request: Request,+ * autoscaledPool: AutoscaledPool,+ * session: Session,+ * $: Cheerio,+ * body: String|Buffer,+ * json: Object,+ * request: Request,+ * contentType: Object,+ * response: Object,+ * proxyInfo: ProxyInfo* }* ```* where the {@link Request} instance corresponds to the failed request, and the `Error` instance@@ -464,22 +473,23 @@ class CheerioCrawler {/*** Wrapper around handlePageFunction that opens and closes pages etc.*- * @param {Object} options- * @param {Request} options.request- * @param {AutoscaledPool} options.autoscaledPool- * @param {Session} [options.session]+ * @param {Object} crawlingContext+ * @param {Request} crawlingContext.request+ * @param {AutoscaledPool} crawlingContext.autoscaledPool+ * @param {Session} [crawlingContext.session]* @ignore*/- async _handleRequestFunction({ request, autoscaledPool, session }) {- let proxyInfo;- let proxyUrl;+ async _handleRequestFunction(crawlingContext) {+ const { request, session } = crawlingContext;+if (this.proxyConfiguration) {- proxyInfo = this.proxyConfiguration.newProxyInfo(session ? session.id : undefined);- proxyUrl = proxyInfo.url;+ const sessionId = session ? session.id : undefined;+ crawlingContext.proxyInfo = this.proxyConfiguration.newProxyInfo(sessionId);}- if (this.prepareRequestFunction) await this.prepareRequestFunction({ request, session, proxyInfo });+ if (this.prepareRequestFunction) await this.prepareRequestFunction(crawlingContext);+ const proxyUrl = crawlingContext.proxyInfo && crawlingContext.proxyInfo.url;const { dom, isXml, body, contentType, response } = await addTimeoutToPromise(this._requestFunction({ request, session, proxyUrl }),this.requestTimeoutMillis,@@ -495,23 +505,21 @@ class CheerioCrawler {}request.loadedUrl = response.url;- const { log } = this;const $ = dom ? cheerio.load(dom, { xmlMode: isXml }) : null;- const context = {- $,- // Using a getter here not to break the original API- // and lazy load the HTML only when needed.- get html() {- log.deprecated('The "html" parameter of handlePageFunction is deprecated, use "body" instead.');- return dom && !isXml && $.html({ decodeEntities: false });- },- get json() {++ crawlingContext.$ = $;+ crawlingContext.contentType = contentType;+ crawlingContext.response = response;+ Object.defineProperty(crawlingContext, 'json', {+ get() {if (contentType.type !== 'application/json') return null;const jsonString = body.toString(contentType.encoding);return JSON.parse(jsonString);},- get body() {+ });+ Object.defineProperty(crawlingContext, 'body', {+ get() {// NOTE: For XML/HTML documents, we don't store the original body and only reconstruct it from Cheerio's DOM.// This is to save memory for high-concurrency crawls. The downside is that changes// made to DOM are reflected in the HTML, but we can live with that...@@ -520,15 +528,10 @@ class CheerioCrawler {}return body;},- contentType,- request,- response,- autoscaledPool,- session,- proxyInfo,- };+ });+return addTimeoutToPromise(- this.handlePageFunction(context),+ this.handlePageFunction(crawlingContext),this.handlePageTimeoutMillis,`handlePageFunction timed out after ${this.handlePageTimeoutMillis / 1000} seconds.`,);@@ -741,6 +744,7 @@ export default CheerioCrawler;* Original instance fo the {Request} object. Must be modified in-place.* @property {Session} [session]* The current session+ * @property {AutoscaledPool} autoscaledPool* @property {ProxyInfo} [proxyInfo]* An object with information about currently used proxy by the crawler* and configured by the {@link ProxyConfiguration} class.diff --git a/src/crawlers/puppeteer_crawler.js b/src/crawlers/puppeteer_crawler.jsindex 1259634cb..6f563e0e9 100644--- a/src/crawlers/puppeteer_crawler.js+++ b/src/crawlers/puppeteer_crawler.js@@ -88,8 +88,14 @@ import { validators } from '../validators';* The function receives the following object as an argument:* ```* {- * request: Request,* error: Error,+ * request: Request,+ * response: Response,+ * page: Page,+ * puppeteerPool: PuppeteerPool,+ * autoscaledPool: AutoscaledPool,+ * session: Session,+ * proxyInfo: ProxyInfo,* }* ```* Where the {@link Request} instance corresponds to the failed request, and the `Error` instance@@ -365,37 +371,37 @@ class PuppeteerCrawler {/*** Wrapper around handlePageFunction that opens and closes pages etc.*- * @param {Object} options- * @param {Request} options.request- * @param {AutoscaledPool} options.autoscaledPool+ * @param {Object} crawlingContext+ * @param {Request} crawlingContext.request+ * @param {AutoscaledPool} crawlingContext.autoscaledPool+ * @param {Session} [crawlingContext.session]* @ignore*/- async _handleRequestFunction({ request, autoscaledPool }) {- let session;- let proxyInfo;- const page = await this.puppeteerPool.newPage();+ async _handleRequestFunction(crawlingContext) {+ crawlingContext.page = await this.puppeteerPool.newPage();+ const { page, request } = crawlingContext;// eslint-disable-next-line no-underscore-dangleconst browserInstance = this.puppeteerPool._getBrowserInstance(page);if (this.sessionPool) {- // eslint-disable-next-line prefer-destructuring- session = browserInstance.session;+ crawlingContext.session = browserInstance.session;// setting cookies to pageif (this.persistCookiesPerSession) {- await page.setCookie(...session.getPuppeteerCookies(request.url));+ await page.setCookie(...crawlingContext.session.getPuppeteerCookies(request.url));}}+ const { session } = crawlingContext;+if (this.proxyConfiguration) {- // eslint-disable-next-line prefer-destructuring- proxyInfo = browserInstance.proxyInfo;+ crawlingContext.proxyInfo = browserInstance.proxyInfo;}try {let response;try {- response = await this.gotoFunction({ page, request, autoscaledPool, puppeteerPool: this.puppeteerPool, session, proxyInfo });+ response = await this.gotoFunction(crawlingContext);} catch (err) {// It would be better to compare the instances,// but we don't have access to puppeteer.errors here.@@ -421,8 +427,11 @@ class PuppeteerCrawler {session.setPuppeteerCookies(cookies, request.loadedUrl);}+ crawlingContext.response = response;+ crawlingContext.puppeteerPool = this.puppeteerPool;+await addTimeoutToPromise(- this.handlePageFunction({ page, request, autoscaledPool, puppeteerPool: this.puppeteerPool, response, session, proxyInfo }),+ this.handlePageFunction(crawlingContext),this.handlePageTimeoutMillis,`handlePageFunction timed out after ${this.handlePageTimeoutMillis / 1000} seconds.`,);@@ -437,6 +446,10 @@ class PuppeteerCrawler {* @param {Object} options* @param {PuppeteerPage} options.page* @param {Request} options.request+ * @property {AutoscaledPool} autoscaledPool+ * @property {PuppeteerPool} puppeteerPool+ * @property {Session} [session]+ * @property {ProxyInfo} [proxyInfo]* @return {Promise<PuppeteerResponse>}* @ignore*/@@ -522,7 +535,7 @@ export default PuppeteerCrawler;* @property {AutoscaledPool} autoscaledPool An instance of the `AutoscaledPool`.* @property {PuppeteerPool} puppeteerPool An instance of the {@link PuppeteerPool} used by this `PuppeteerCrawler`.* @property {Session} [session] `Session` object for this request.- * @property {ProxyInfo} [proxyInfo]+ * @property {ProxyInfo} [proxyInfo] Proxy info object*//**diff --git a/test/crawlers/cheerio_crawler.test.js b/test/crawlers/cheerio_crawler.test.jsindex 35470b803..5f0709084 100644--- a/test/crawlers/cheerio_crawler.test.js+++ b/test/crawlers/cheerio_crawler.test.js@@ -14,7 +14,9 @@ import { STATUS_CODES_BLOCKED } from '../../build/constants';import LocalStorageDirEmulator from '../local_storage_dir_emulator';import * as utilsRequest from '../../build/utils_request';import CrawlerExtension from '../../build/crawlers/crawler_extension';+import Request from '../../build/request';import * as utils from '../../build/utils';+import AutoscaledPool from '../../build/autoscaling/autoscaled_pool';// Add common props to mocked request responses.const responseMock = {@@ -112,9 +114,9 @@ describe('CheerioCrawler', () => {const requestList = await getRequestListForMirror(port);const processed = [];const failed = [];- const handlePageFunction = async ({ $, html, request }) => {+ const handlePageFunction = async ({ $, body, request }) => {request.userData.title = $('title').text();- request.userData.html = html;+ request.userData.body = body;processed.push(request);};@@ -134,8 +136,8 @@ describe('CheerioCrawler', () => {processed.forEach((request) => {expect(request.userData.title).toBe('Title');- expect(typeof request.userData.html).toBe('string');- expect(request.userData.html.length).not.toBe(0);+ expect(typeof request.userData.body).toBe('string');+ expect(request.userData.body.length).not.toBe(0);});});@@ -906,6 +908,95 @@ describe('CheerioCrawler', () => {});});+ describe('Crawling context', () => {+ const sources = ['http://example.com/'];+ let requestList;+ let actualLogLevel;+ beforeEach(async () => {+ actualLogLevel = log.getLevel();+ log.setLevel(log.LEVELS.OFF);+ requestList = await Apify.openRequestList(null, sources.slice());+ });++ afterAll(() => {+ log.setLevel(actualLogLevel);+ });++ test('uses correct crawling context', async () => {+ let prepareCrawlingContext;++ const prepareRequestFunction = async (crawlingContext) => {+ prepareCrawlingContext = crawlingContext;+ expect(crawlingContext.request).toBeInstanceOf(Request);+ expect(crawlingContext.autoscaledPool).toBeInstanceOf(AutoscaledPool);+ expect(crawlingContext.session).toBeInstanceOf(Session);+ };++ const handlePageFunction = async (crawlingContext) => {+ expect(crawlingContext === prepareCrawlingContext).toEqual(true);+ expect(crawlingContext.request).toBeInstanceOf(Request);+ expect(crawlingContext.autoscaledPool).toBeInstanceOf(AutoscaledPool);+ expect(crawlingContext.session).toBeInstanceOf(Session);+ expect(typeof crawlingContext.$).toBe('function');+ expect(typeof crawlingContext.response).toBe('object');+ expect(typeof crawlingContext.contentType).toBe('object');++ throw new Error('some error');+ };++ const handleFailedRequestFunction = async (crawlingContext) => {+ expect(crawlingContext === prepareCrawlingContext).toEqual(true);+ expect(crawlingContext.request).toBeInstanceOf(Request);+ expect(crawlingContext.autoscaledPool).toBeInstanceOf(AutoscaledPool);+ expect(crawlingContext.session).toBeInstanceOf(Session);+ expect(typeof crawlingContext.$).toBe('function');+ expect(typeof crawlingContext.response).toBe('object');+ expect(typeof crawlingContext.contentType).toBe('object');++ expect(crawlingContext.error).toBeInstanceOf(Error);+ expect(crawlingContext.error.message).toEqual('some error');+ };++ const cheerioCrawler = new Apify.CheerioCrawler({+ requestList,+ maxRequestRetries: 0,+ maxConcurrency: 1,+ useSessionPool: true,+ prepareRequestFunction,+ handlePageFunction,+ handleFailedRequestFunction,+ });+ await cheerioCrawler.run();+ });++ test('handleFailedRequestFunction contains proxyInfo', async () => {+ process.env[ENV_VARS.PROXY_PASSWORD] = 'abc123';++ const stub = sinon.stub(utilsRequest, 'requestAsBrowser').resolves({ body: { connected: true } });++ const proxyConfiguration = await Apify.createProxyConfiguration();++ const cheerioCrawler = new Apify.CheerioCrawler({+ requestList,+ maxRequestRetries: 0,+ maxConcurrency: 1,+ proxyConfiguration,+ handlePageFunction: async () => {+ throw new Error('some error');+ },+ handleFailedRequestFunction: async (crawlingContext) => {+ expect(typeof crawlingContext.proxyInfo).toEqual('object');+ expect(crawlingContext.proxyInfo.hasOwnProperty('url')).toEqual(true);+ },+ useSessionPool: true,+ });+ await cheerioCrawler.run();++ delete process.env[ENV_VARS.PROXY_PASSWORD];+ stub.restore();+ });+ });+describe('use', () => {const sources = ['http://example.com/'];let requestList;diff --git a/test/crawlers/puppeteer_crawler.test.js b/test/crawlers/puppeteer_crawler.test.jsindex f717504d6..e21026172 100644--- a/test/crawlers/puppeteer_crawler.test.js+++ b/test/crawlers/puppeteer_crawler.test.js@@ -6,6 +6,10 @@ import { STATUS_CODES_BLOCKED } from '../../build/constants';import LocalStorageDirEmulator from '../local_storage_dir_emulator';import * as utilsRequest from '../../build/utils_request';import * as utils from '../../build/utils';+import Request from '../../build/request';+import AutoscaledPool from '../../build/autoscaling/autoscaled_pool';+import { Session } from '../../build/session_pool/session';+import PuppeteerPool from '../../build/puppeteer_pool';describe('PuppeteerCrawler', () => {let prevEnvHeadless;@@ -426,4 +430,95 @@ describe('PuppeteerCrawler', () => {delete process.env[ENV_VARS.PROXY_PASSWORD];});});++ describe('Crawling context', () => {+ const sources = ['http://example.com/'];+ let requestList;+ let actualLogLevel;+ beforeEach(async () => {+ actualLogLevel = log.getLevel();+ log.setLevel(log.LEVELS.OFF);+ requestList = await Apify.openRequestList(null, sources.slice());+ });++ afterAll(() => {+ log.setLevel(actualLogLevel);+ });++ test('uses correct crawling context', async () => {+ let prepareCrawlingContext;++ const gotoFunction = async (crawlingContext) => {+ prepareCrawlingContext = crawlingContext;+ expect(crawlingContext.request).toBeInstanceOf(Request);+ expect(crawlingContext.autoscaledPool).toBeInstanceOf(AutoscaledPool);+ expect(crawlingContext.session).toBeInstanceOf(Session);+ expect(typeof crawlingContext.page).toBe('object');+ };++ const handlePageFunction = async (crawlingContext) => {+ expect(crawlingContext === prepareCrawlingContext).toEqual(true);+ expect(crawlingContext.request).toBeInstanceOf(Request);+ expect(crawlingContext.autoscaledPool).toBeInstanceOf(AutoscaledPool);+ expect(crawlingContext.session).toBeInstanceOf(Session);+ expect(typeof crawlingContext.page).toBe('object');+ expect(crawlingContext.puppeteerPool).toBeInstanceOf(PuppeteerPool);+ expect(crawlingContext.hasOwnProperty('response')).toBe(true);++ throw new Error('some error');+ };++ const handleFailedRequestFunction = async (crawlingContext) => {+ expect(crawlingContext === prepareCrawlingContext).toEqual(true);+ expect(crawlingContext.request).toBeInstanceOf(Request);+ expect(crawlingContext.autoscaledPool).toBeInstanceOf(AutoscaledPool);+ expect(crawlingContext.session).toBeInstanceOf(Session);+ expect(typeof crawlingContext.page).toBe('object');+ expect(crawlingContext.puppeteerPool).toBeInstanceOf(PuppeteerPool);+ expect(crawlingContext.hasOwnProperty('response')).toBe(true);++ expect(crawlingContext.error).toBeInstanceOf(Error);+ expect(crawlingContext.error.message).toEqual('some error');+ };++ const puppeteerCrawler = new Apify.PuppeteerCrawler({+ requestList,+ maxRequestRetries: 0,+ maxConcurrency: 1,+ useSessionPool: true,+ gotoFunction,+ handlePageFunction,+ handleFailedRequestFunction,+ });++ await puppeteerCrawler.run();+ });++ test('handleFailedRequestFunction contains proxyInfo', async () => {+ process.env[ENV_VARS.PROXY_PASSWORD] = 'abc123';+ const stub = sinon.stub(utilsRequest, 'requestAsBrowser').resolves({ body: { connected: true } });++ const proxyConfiguration = await Apify.createProxyConfiguration();++ const puppeteerCrawler = new Apify.PuppeteerCrawler({+ requestList,+ maxRequestRetries: 0,+ maxConcurrency: 1,+ useSessionPool: true,+ proxyConfiguration,+ handlePageFunction: async () => {+ throw new Error('some error');+ },+ handleFailedRequestFunction: async (crawlingContext) => {+ expect(typeof crawlingContext.proxyInfo).toEqual('object');+ expect(crawlingContext.proxyInfo.hasOwnProperty('url')).toEqual(true);+ },+ });++ await puppeteerCrawler.run();++ delete process.env[ENV_VARS.PROXY_PASSWORD];+ stub.restore();+ });+ });});