Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
refactored code.
Petr Patek0bcdecb60bfb
diff --git a/src/constants.js b/src/constants.jsindex 8a2947aed..1fd91d23a 100644--- a/src/constants.js+++ b/src/constants.js@@ -126,4 +126,4 @@ export const COUNTRY_CODE_REGEX = /^[A-Z]{2}$/;export const STATUS_CODES_BLOCKED = [401, 403, 429];-export const BROWSER_SESSION_ID_KEY_NAME = 'APIFY_SESSION_ID';+export const BROWSER_SESSION_KEY_NAME = 'APIFY_SESSION';diff --git a/src/puppeteer_pool.js b/src/puppeteer_pool.jsindex 82026b449..f1510bb16 100644--- a/src/puppeteer_pool.js+++ b/src/puppeteer_pool.js@@ -11,7 +11,7 @@ import { launchPuppeteer } from './puppeteer';import { addTimeoutToPromise } from './utils';import LiveViewServer from './live_view/live_view_server';import EVENTS from './session_pool/events';-import { BROWSER_SESSION_ID_KEY_NAME } from './constants';+import { BROWSER_SESSION_KEY_NAME } from './constants';const PROCESS_KILL_TIMEOUT_MILLIS = 5000;@@ -254,7 +254,7 @@ class PuppeteerPool {}browser.recycleDiskCacheDir = diskCacheDir;- if (session) browser[BROWSER_SESSION_ID_KEY_NAME] = session;+ if (session) browser[BROWSER_SESSION_KEY_NAME] = session;return browser;};@@ -322,7 +322,7 @@ class PuppeteerPool {browser = await browserPromise;if (this.sessionPool) {- instance.session = browser[BROWSER_SESSION_ID_KEY_NAME];+ instance.session = browser[BROWSER_SESSION_KEY_NAME];}} catch (err) {log.exception(err, 'PuppeteerPool: Browser launch failed', { id });diff --git a/test/puppeteer_pool.test.js b/test/puppeteer_pool.test.jsindex f7b42eb3c..36221e293 100644--- a/test/puppeteer_pool.test.js+++ b/test/puppeteer_pool.test.js@@ -6,7 +6,7 @@ import * as Apify from '../build/index';import { launchPuppeteer } from '../build/puppeteer';import { SessionPool } from '../build/session_pool/session_pool';import { LOCAL_STORAGE_DIR, emptyLocalStorageSubdir } from './_helper';-import { BROWSER_SESSION_ID_KEY_NAME } from '../src/constants';+import { BROWSER_SESSION_KEY_NAME } from '../src/constants';import { sleep } from '../src/utils';@@ -647,9 +647,9 @@ describe('PuppeteerPool', () => {expect(pool.sessionPool.constructor.name).toEqual('SessionPool');const page = await pool.newPage();const browser = page.browser();- expect(browser[BROWSER_SESSION_ID_KEY_NAME].id).toEqual(sessionPool.sessions[0].id);+ expect(browser[BROWSER_SESSION_KEY_NAME].id).toEqual(sessionPool.sessions[0].id);expect(- Object.values(pool.activeInstances).filter(instance => instance.session.id === browser[BROWSER_SESSION_ID_KEY_NAME].id),+ Object.values(pool.activeInstances).filter(instance => instance.session.id === browser[BROWSER_SESSION_KEY_NAME].id),).toHaveLength(1);sessionPool.sessions[0].retire();