Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
fix: proxy per page broke session fetching (#1256)
Szymon Marczakb0a5bc056cff
diff --git a/src/crawlers/browser_crawler.js b/src/crawlers/browser_crawler.jsindex 65a57bfc6..93413c824 100644--- a/src/crawlers/browser_crawler.js+++ b/src/crawlers/browser_crawler.js@@ -360,14 +360,14 @@ export default class BrowserCrawler extends BasicCrawler {* @internal*/async _handleRequestFunction(crawlingContext) {- const { request, session } = crawlingContext;-const newPageOptions = {id: crawlingContext.id,};const useIncognitoPages = this.launchContext && this.launchContext.useIncognitoPages;if (this.proxyConfiguration && useIncognitoPages) {+ const { session } = crawlingContext;+const proxyInfo = this.proxyConfiguration.newProxyInfo(session && session.id);crawlingContext.session = session;crawlingContext.proxyInfo = proxyInfo;@@ -390,6 +390,10 @@ export default class BrowserCrawler extends BasicCrawler {tryCancel();this._enhanceCrawlingContextWithPageInfo(crawlingContext, page, useIncognitoPages);+ // DO NOT MOVE THIS LINE ABOVE!+ // `enhanceCrawlingContextWithPageInfo` can (and will!) override those properties.+ const { request, session } = crawlingContext;+if (this.useSessionPool) {const sessionCookies = session.getPuppeteerCookies(request.url);if (sessionCookies.length) {