Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
fix: respect config object when creating `SessionPool` (#1881)
Martin Adámekdb069df80bc1
diff --git a/packages/basic-crawler/src/internals/basic-crawler.ts b/packages/basic-crawler/src/internals/basic-crawler.tsindex a0b17e9a3..f60ca0b69 100644--- a/packages/basic-crawler/src/internals/basic-crawler.ts+++ b/packages/basic-crawler/src/internals/basic-crawler.ts@@ -842,7 +842,7 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContextthis.autoscaledPool = new AutoscaledPool(this.autoscaledPoolOptions, this.config);if (this.useSessionPool) {- this.sessionPool = await SessionPool.open(this.sessionPoolOptions);+ this.sessionPool = await SessionPool.open(this.sessionPoolOptions, this.config);// Assuming there are not more than 20 browsers running at once;this.sessionPool.setMaxListeners(20);}diff --git a/packages/core/src/session_pool/session_pool.ts b/packages/core/src/session_pool/session_pool.tsindex 957eea698..8fcaac689 100644--- a/packages/core/src/session_pool/session_pool.ts+++ b/packages/core/src/session_pool/session_pool.ts@@ -437,8 +437,8 @@ export class SessionPool extends EventEmitter {** For more details and code examples, see the {@apilink SessionPool} class.*/- static async open(options?: SessionPoolOptions): Promise<SessionPool> {- const sessionPool = new SessionPool(options);+ static async open(options?: SessionPoolOptions, config?: Configuration): Promise<SessionPool> {+ const sessionPool = new SessionPool(options, config);await sessionPool.initialize();return sessionPool;}