Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
implementation proposal.
Petr Patek2327f35cdd52
diff --git a/src/constants.js b/src/constants.jsindex 15ffef8d6..8a2947aed 100644--- a/src/constants.js+++ b/src/constants.js@@ -125,3 +125,5 @@ export const BASIC_CRAWLER_TIMEOUT_MULTIPLIER = 10;export const COUNTRY_CODE_REGEX = /^[A-Z]{2}$/;export const STATUS_CODES_BLOCKED = [401, 403, 429];++export const BROWSER_SESSION_ID_KEY_NAME = 'APIFY_SESSION_ID';diff --git a/src/crawlers/puppeteer_crawler.js b/src/crawlers/puppeteer_crawler.jsindex ec5eae267..d94ccbb65 100644--- a/src/crawlers/puppeteer_crawler.js+++ b/src/crawlers/puppeteer_crawler.js@@ -4,8 +4,9 @@ import _ from 'underscore';import BasicCrawler from './basic_crawler';import PuppeteerPool from '../puppeteer_pool';import { addTimeoutToPromise } from '../utils';-import { BASIC_CRAWLER_TIMEOUT_MULTIPLIER } from '../constants';+import { BASIC_CRAWLER_TIMEOUT_MULTIPLIER, BROWSER_SESSION_ID_KEY_NAME } from '../constants';import { gotoExtended } from '../puppeteer_utils';+import { openSessionPool } from '../session_pool/session_pool';/*** Provides a simple framework for parallel crawling of web pages@@ -197,6 +198,10 @@ class PuppeteerCrawler {puppeteerPoolOptions,launchPuppeteerFunction,launchPuppeteerOptions,++ sessionPoolOptions,+ persistCookiesPerSession,+ useSessionPool,} = options;checkParamOrThrow(handlePageFunction, 'options.handlePageFunction', 'Function');@@ -205,6 +210,9 @@ class PuppeteerCrawler {checkParamOrThrow(gotoFunction, 'options.gotoFunction', 'Function');checkParamOrThrow(gotoTimeoutSecs, 'options.gotoTimeoutSecs', 'Number');checkParamOrThrow(puppeteerPoolOptions, 'options.puppeteerPoolOptions', 'Maybe Object');+ checkParamOrThrow(useSessionPool, 'options.useSessionPool', 'Boolean');+ checkParamOrThrow(sessionPoolOptions, 'options.sessionPoolOptions', 'Object');+ checkParamOrThrow(persistCookiesPerSession, 'options.persistCookiesPerSession', 'Boolean');if (options.gotoTimeoutSecs && options.gotoFunction) {log.warning('PuppeteerCrawler: You are using gotoTimeoutSecs with a custom gotoFunction. '@@ -224,6 +232,8 @@ class PuppeteerCrawler {};this.puppeteerPool = null; // Constructed when .run()+ this.useSessionPool = useSessionPool;+ this.sessionPoolOptions = sessionPoolOptions;this.basicCrawler = new BasicCrawler({// Basic crawler options.@@ -250,11 +260,18 @@ class PuppeteerCrawler {async run() {if (this.isRunningPromise) return this.isRunningPromise;+ if (this.useSessionPool) {+ this.sessionPool = await openSessionPool(this.sessionPoolOptions);+ this.puppeteerPoolOptions.sessionPool = this.sessionPool;+ }this.puppeteerPool = new PuppeteerPool(this.puppeteerPoolOptions);try {this.isRunningPromise = this.basicCrawler.run();await this.isRunningPromise;} finally {+ if (this.useSessionPool) {+ this.sessionPool.teardown();+ }this.puppeteerPool.destroy();}}@@ -265,17 +282,27 @@ class PuppeteerCrawler {* @ignore*/async _handleRequestFunction({ request, autoscaledPool }) {+ let session;const page = await this.puppeteerPool.newPage();+ if (this.sessionPool) {+ // @TODO: Handle cookies load here.+ const browser = page.browser();+ session = this.sessionPool.sessions.find(ses => ses.id === browser[BROWSER_SESSION_ID_KEY_NAME]);+ }+try {- const response = await this.gotoFunction({ page, request, autoscaledPool, puppeteerPool: this.puppeteerPool });+ const response = await this.gotoFunction({ page, request, autoscaledPool, puppeteerPool: this.puppeteerPool, session });await this.puppeteerPool.serveLiveViewSnapshot(page);+ // @TODO: Handle cookies save here.request.loadedUrl = page.url();await addTimeoutToPromise(- this.handlePageFunction({ page, request, autoscaledPool, puppeteerPool: this.puppeteerPool, response }),+ this.handlePageFunction({ page, request, autoscaledPool, puppeteerPool: this.puppeteerPool, response, session }),this.handlePageTimeoutMillis,`PuppeteerCrawler: handlePageFunction timed out after ${this.handlePageTimeoutMillis / 1000} seconds.`,);++ if (session) session.markGood();} finally {await this.puppeteerPool.recyclePage(page);}diff --git a/src/puppeteer_pool.js b/src/puppeteer_pool.jsindex b549da60b..27b078f98 100644--- a/src/puppeteer_pool.js+++ b/src/puppeteer_pool.js@@ -10,6 +10,9 @@ import { checkParamOrThrow } from 'apify-client/build/utils';import { launchPuppeteer } from './puppeteer';import { addTimeoutToPromise } from './utils';import LiveViewServer from './live_view/live_view_server';+import EVENTS from './session_pool/events';+import { BROWSER_SESSION_ID_KEY_NAME } from './constants';+const PROCESS_KILL_TIMEOUT_MILLIS = 5000;const PAGE_CLOSE_KILL_TIMEOUT_MILLIS = 1000;@@ -175,6 +178,7 @@ class PuppeteerPool {useIncognitoPages,proxyUrls,useLiveView,+ sessionPool,} = _.defaults({}, options, DEFAULT_OPTIONS);// Disabling due to memory leak.@@ -202,8 +206,10 @@ class PuppeteerPool {// Enforce non-empty proxyUrls arrayif (proxyUrls && !proxyUrls.length) throw new Error('Parameter "options.proxyUrls" of type Array must not be empty');checkParamOrThrow(useLiveView, 'options.useLiveView', 'Maybe Boolean');+ checkParamOrThrow(sessionPool, 'options.sessionPool', 'Maybe Object');// Config.+ this.sessionPool = sessionPool;this.reusePages = reusePages;this.maxOpenPagesPerInstance = maxOpenPagesPerInstance;this.retireInstanceAfterRequestCount = retireInstanceAfterRequestCount;@@ -235,12 +241,21 @@ class PuppeteerPool {// Set timeout for browser launch.if (opts.timeout == null) opts.timeout = this.puppeteerOperationTimeoutMillis;+ let session;+ if (sessionPool) {+ session = await sessionPool.getSession();+ opts.apifyProxySession = session.id;+ }+const browser = await launchPuppeteerFunction(opts);if (!browser || typeof browser.newPage !== 'function') {// eslint-disable-next-line max-lenthrow new Error("The custom 'launchPuppeteerFunction' passed to PuppeteerPool must return a promise resolving to Puppeteer's Browser instance.");}browser.recycleDiskCacheDir = diskCacheDir;++ if (session) browser[BROWSER_SESSION_ID_KEY_NAME] = session.id;+return browser;};@@ -266,6 +281,11 @@ class PuppeteerPool {// ensure termination on SIGINTthis.sigintListener = () => this._killAllInstances();process.on('SIGINT', this.sigintListener);++ if (sessionPool) {+ this._retireBrowserWithSession = this._retireBrowserWithSession.bind(this);+ sessionPool.on(EVENTS.SESSION_RETIRED, this._retireBrowserWithSession);+ }}/**@@ -300,6 +320,10 @@ class PuppeteerPool {let browser;try {browser = await browserPromise;++ if (this.sessionPool) {+ instance.sessionId = browser[BROWSER_SESSION_ID_KEY_NAME];+ }} catch (err) {log.exception(err, 'PuppeteerPool: Browser launch failed', { id });delete this.activeInstances[id];@@ -705,6 +729,28 @@ class PuppeteerPool {.catch(err => log.debug('Live View failed to be served.', { message: err.message }));this.liveViewSnapshotsInProgress.set(page, snapshotPromise);}++ _findInstancesBySession(session) {+ const instances = Object.values(this.activeInstances);+ return instances.filter(instance => instance.sessionId === session.id);+ }++ async _retireBrowserWithSession(session) {+ const instances = this._findInstancesBySession(session);++ const isInstanceRunning = instances.length >= 1;++ if (isInstanceRunning) {+ const retireInstances = instances.map(instance => this._retireInstance(instance));+ try {+ await Promise.all(retireInstances);+ } catch (e) {+ // ignore the error, since the instance is either retired already, being retired or cannot be retired+ // ( we cant do nothing about this at this point)+ log.debug('Could not retire instances ', e);+ }+ }+ }}export default PuppeteerPool;diff --git a/src/session_pool/session_pool.js b/src/session_pool/session_pool.jsindex b8505835b..e33f4428b 100644--- a/src/session_pool/session_pool.js+++ b/src/session_pool/session_pool.js@@ -25,7 +25,7 @@ import { ACTOR_EVENT_NAMES_EX } from '../constants';* sessionOptions:{* maxAgeSecs: 10,* maxUsageCount: 150, // for example when you know that the site blocks after 150 requests.- * }+ * },* persistStateKeyValueStoreId: 'my-key-value-store-for-sessions',* persistStateKey: 'my-session-pool',* });@@ -83,7 +83,7 @@ export class SessionPool extends EventEmitter {// ValidationcheckParamOrThrow(maxPoolSize, 'options.maxPoolSize', 'Number');- checkParamOrThrow(sessionOptions, 'options.maxAgeSecs', 'Object');+ checkParamOrThrow(sessionOptions, 'options.sessionOptions', 'Object');checkParamOrThrow(persistStateKeyValueStoreId, 'options.persistStateKeyValueStoreId', 'Maybe String');checkParamOrThrow(persistStateKey, 'options.persistStateKey', 'String');checkParamOrThrow(createSessionFunction, 'options.createSessionFunction', 'Maybe Function');diff --git a/test/puppeteer_pool.test.js b/test/puppeteer_pool.test.jsindex d92a94193..70a3653c4 100644--- a/test/puppeteer_pool.test.js+++ b/test/puppeteer_pool.test.js@@ -4,6 +4,11 @@ import log from 'apify-shared/log';import { ENV_VARS } from 'apify-shared/consts';import * as Apify from '../build/index';import { launchPuppeteer } from '../build/puppeteer';+import { SessionPool } from '../build/session_pool/session_pool';+import { LOCAL_STORAGE_DIR, emptyLocalStorageSubdir } from './_helper';+import { BROWSER_SESSION_ID_KEY_NAME } from '../src/constants';+import { sleep } from '../src/utils';+const shortSleep = (millis = 25) => new Promise(resolve => setTimeout(resolve, millis));@@ -621,4 +626,36 @@ describe('PuppeteerPool', () => {expect(stopped).toBe(1);});});++ describe('uses sessionPool', () => {+ beforeAll(() => {+ process.env.APIFY_LOCAL_STORAGE_DIR = LOCAL_STORAGE_DIR;+ });++ afterEach(() => {+ emptyLocalStorageSubdir('key_value_stores/default');+ });++ test('should work', async () => {+ const sessionPool = new SessionPool();+ await sessionPool.initialize();+ const pool = new Apify.PuppeteerPool({+ launchPuppeteerOptions: { headless: true },+ useLiveView: true,+ sessionPool,+ });+ expect(pool.sessionPool.constructor.name).toEqual('SessionPool');+ const page = await pool.newPage();+ const browser = page.browser();+ expect(browser[BROWSER_SESSION_ID_KEY_NAME]).toEqual(sessionPool.sessions[0].id);+ expect(+ Object.values(pool.activeInstances).filter(instance => instance.sessionId === browser[BROWSER_SESSION_ID_KEY_NAME]),+ ).toHaveLength(1);++ sessionPool.sessions[0].retire();++ await sleep(2000);+ expect(Object.values(pool.activeInstances)).toHaveLength(0);+ });+ });});