Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Fix leading dot puppeteer cookies in Session
Ondra Urban91dfc4cb69ae
diff --git a/src/session_pool/session.js b/src/session_pool/session.jsindex 4bb136159..6a08f8d1f 100644--- a/src/session_pool/session.js+++ b/src/session_pool/session.js@@ -119,6 +119,8 @@ export class Session {this.errorScore = errorScore; // indicates number of markBaded request with the sessionthis.maxUsageCount = maxUsageCount;this.sessionPool = sessionPool;++ this._puppeteerCookieToTough = this._puppeteerCookieToTough.bind(this);}/**@@ -267,12 +269,8 @@ export class Session {* @param {string} url*/setPuppeteerCookies(cookies, url) {- try {- this._setCookies(cookies.map(this._puppeteerCookieToTough.bind(this)), url);- } catch (e) {- // if invalid cookies are provided just log the exception. No need to retry the request automatically.- this.log.exception(e, 'Could not set cookies in puppeteer format.');- }+ const normalizedCookies = cookies.map(this._puppeteerCookieToTough);+ this._setCookies(normalizedCookies, url);}/**@@ -306,11 +304,14 @@ export class Session {_puppeteerCookieToTough(puppeteerCookie) {const isExpiresValid = puppeteerCookie.expires && typeof puppeteerCookie.expires === 'number';const expires = isExpiresValid ? new Date(puppeteerCookie.expires * 1000) : this._getDefaultCookieExpirationDate(this.maxAgeSecs);+ const domain = typeof puppeteerCookie.domain === 'string' && puppeteerCookie.domain.startsWith('.')+ ? puppeteerCookie.domain.slice(1)+ : puppeteerCookie.domain;return new Cookie({key: puppeteerCookie.name,value: puppeteerCookie.value,expires,- domain: puppeteerCookie.domain,+ domain,path: puppeteerCookie.path,secure: puppeteerCookie.secure,httpOnly: puppeteerCookie.httpOnly,@@ -342,9 +343,16 @@ export class Session {* @private*/_setCookies(cookies, url) {+ const errorMessages = [];for (const cookie of cookies) {- this.cookieJar.setCookieSync(cookie, url, { ignoreError: false });+ try {+ this.cookieJar.setCookieSync(cookie, url, { ignoreError: false });+ } catch (e) {+ errorMessages.push(e.message);+ }}+ // if invalid cookies are provided just log the exception. No need to retry the request automatically.+ this.log.error('Could not set cookies.', { errorMessages });}/**diff --git a/test/session_pool/session.test.js b/test/session_pool/session.test.jsindex 099ec4d5b..0d8a6f92b 100644--- a/test/session_pool/session.test.js+++ b/test/session_pool/session.test.js@@ -26,19 +26,16 @@ describe('Session - testing session behaviour ', () => {expect(session.errorScore).toBe(0.5);});- test(- 'should throw error when param sessionPool is not EventEmitter instance',- () => {- let err;- try {+ test('should throw error when param sessionPool is not EventEmitter instance', () => {+ let err;+ try {const session = new Session({ sessionPool: {} }); // eslint-disable-line- } catch (e) {- err = e;- }- expect(err).toBeDefined(); // eslint-disable-line- expect(err.message.includes('sessionPool must be instance of SessionPool')).toBe(true); // eslint-disable-line- },- );+ } catch (e) {+ err = e;+ }+ expect(err).toBeDefined(); // eslint-disable-line+ expect(err.message.includes('sessionPool must be instance of SessionPool')).toBe(true); // eslint-disable-line+ });test('should mark session markBad', () => {session.markBad();@@ -176,6 +173,18 @@ describe('Session - testing session behaviour ', () => {expect(session.getCookieString(url)).toBe('cookie1=my-cookie; cookie2=your-cookie');});+ test('setPuppeteerCookies works with leading dots in domains', () => {+ const url = 'https://www.example.com';+ const cookies = [+ { name: 'cookie1', value: 'my-cookie', domain: 'abc.example.com' },+ { name: 'cookie2', value: 'your-cookie', domain: '.example.com' },+ ];++ session = new Session({ sessionPool });+ session.setPuppeteerCookies(cookies, url);+ expect(session.getCookieString(url)).toBe('cookie2=your-cookie');+ });+describe('.putResponse & .getCookieString', () => {test('should set and update cookies from "set-cookie" header', () => {const headers = {};