Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

added cookie logic.

Petr Patek36929314b479
diff --git a/src/crawlers/puppeteer_crawler.js b/src/crawlers/puppeteer_crawler.js
index 1303109e6..c63b79f46 100644
--- a/src/crawlers/puppeteer_crawler.js
+++ b/src/crawlers/puppeteer_crawler.js
@@ -4,7 +4,7 @@ import _ from 'underscore';
import BasicCrawler from './basic_crawler';
import PuppeteerPool from '../puppeteer_pool';
import { addTimeoutToPromise } from '../utils';
-import { BASIC_CRAWLER_TIMEOUT_MULTIPLIER } from '../constants';
+import { BASIC_CRAWLER_TIMEOUT_MULTIPLIER, BROWSER_SESSION_KEY_NAME } from '../constants';
import { gotoExtended } from '../puppeteer_utils';
import { openSessionPool } from '../session_pool/session_pool';
@@ -234,6 +234,7 @@ class PuppeteerCrawler {
this.puppeteerPool = null; // Constructed when .run()
this.useSessionPool = useSessionPool;
this.sessionPoolOptions = sessionPoolOptions;
+ this.persistCookiesPerSession = persistCookiesPerSession;
this.basicCrawler = new BasicCrawler({
// Basic crawler options.
@@ -286,17 +287,26 @@ class PuppeteerCrawler {
const page = await this.puppeteerPool.newPage();
if (this.sessionPool) {
- // @TODO: Handle cookies load here.
const browser = page.browser();
- // @TODO: Session pool method find my id;
- session = browser.session;
+ session = browser[BROWSER_SESSION_KEY_NAME];
+
+ // setting cookies for page
+ if (this.persistCookiesPerSession) {
+ await page.setCookie(...session.getPuppeteerCookies(request.url));
+ }
}
try {
const response = await this.gotoFunction({ page, request, autoscaledPool, puppeteerPool: this.puppeteerPool, session });
await this.puppeteerPool.serveLiveViewSnapshot(page);
- // @TODO: Handle cookies save here.
request.loadedUrl = page.url();
+
+ // save cookies
+ if (this.persistCookiesPerSession) {
+ const cookies = await page.cookies(request.loadedUrl);
+ session.putPuppeteerCookies(cookies, request.loadedUrl);
+ }
+
await addTimeoutToPromise(
this.handlePageFunction({ page, request, autoscaledPool, puppeteerPool: this.puppeteerPool, response, session }),
this.handlePageTimeoutMillis,
diff --git a/src/session_pool/session.js b/src/session_pool/session.js
index 139dd9813..c3b13f0ec 100644
--- a/src/session_pool/session.js
+++ b/src/session_pool/session.js
@@ -6,6 +6,8 @@ import EVENTS from './events';
import { STATUS_CODES_BLOCKED } from '../constants';
import { getCookiesFromResponse } from './session_utils';
+const { Cookie } = tough;
+
/**
* Class aggregating data for session.
@@ -200,11 +202,42 @@ export class Session {
putResponse(response) {
const cookies = getCookiesFromResponse(response).filter(c => c);
+ this.setCookies(cookies, response.url);
+ }
+
+ /**
+ * Persists puppeteer cookies to session for reuse.
+ * @param puppeteerCookies - cookie from puppeteer `page.cookies` method.
+ * @param url - Loaded url from page function.
+ */
+ putPuppeteerCookies(puppeteerCookies, url) {
+ const cookies = puppeteerCookies.map(puppeteerCookie => this._transformPuppeteerCookie(puppeteerCookie));
+
+ this.setCookies(cookies, url);
+ }
+
+ /**
+ * Set cookies to session cookieJar.
+ * Cookies array should be compatible with tough-cookie.
+ * @param cookies {Array<Cookie>}
+ * @param url {String}
+ */
+ setCookies(cookies, url) {
for (const cookie of cookies) {
- this.cookieJar.setCookieSync(cookie, response.url, { ignoreError: true });
+ this.cookieJar.setCookieSync(cookie, url, { ignoreError: false });
}
}
+ /**
+ * Get cookies.
+ * Gets a array of `tough-cookie` Cookie instances.
+ * @param url
+ * @return {Array<Object>}
+ */
+ getCookies(url) {
+ return this.cookieJar.getCookiesSync(url);
+ }
+
/**
* Wrapper around `tough-cookie` Cookie jar `getCookieString` method.
* @param url
@@ -213,4 +246,51 @@ export class Session {
getCookieString(url) {
return this.cookieJar.getCookieStringSync(url, {});
}
+
+ /**
+ * Gets cookies in format ready for puppeteer.
+ * @param url
+ * @return {*}
+ */
+ getPuppeteerCookies(url) {
+ const cookies = this.getCookies(url);
+ return cookies.map(cookie => this._transformToughCookie(cookie));
+ }
+
+
+ /**
+ * Transforms puppeteer cookie to tough-cookie.
+ * @param puppeteerCookie - Cookie from puppeteer `page.cookies` method.
+ * @return {Cookie}
+ * @private
+ */
+ _transformPuppeteerCookie(puppeteerCookie) {
+ return new Cookie({
+ key: puppeteerCookie.name,
+ value: puppeteerCookie.value,
+ expires: new Date(puppeteerCookie.expires),
+ domain: puppeteerCookie.domain,
+ path: puppeteerCookie.path,
+ secure: puppeteerCookie.secure,
+ httpOnly: puppeteerCookie.httpOnly,
+ });
+ }
+
+ /**
+ * Transforms tough-cookie cookie to puppeteer Cookie .
+ * @param toughCookie - Cookie from CookieJar.
+ * @return {Object} - puppeteer cookie
+ * @private
+ */
+ _transformToughCookie(toughCookie) {
+ return {
+ name: toughCookie.key,
+ value: toughCookie.value,
+ expires: new Date(toughCookie.expires).getTime(),
+ domain: toughCookie.domain,
+ path: toughCookie.path,
+ secure: toughCookie.secure,
+ httpOnly: toughCookie.httpOnly,
+ };
+ }
}
diff --git a/test/crawlers/puppeteer_crawler.test.js b/test/crawlers/puppeteer_crawler.test.js
index 6a48cfef2..27c2c3ee3 100644
--- a/test/crawlers/puppeteer_crawler.test.js
+++ b/test/crawlers/puppeteer_crawler.test.js
@@ -1,6 +1,7 @@
import log from 'apify-shared/log';
import { ENV_VARS } from 'apify-shared/consts';
import * as Apify from '../../build';
+import { emptyLocalStorageSubdir, LOCAL_STORAGE_DIR } from '../_helper';
describe('PuppeteerCrawler', () => {
let prevEnvHeadless;
@@ -11,6 +12,11 @@ describe('PuppeteerCrawler', () => {
process.env[ENV_VARS.HEADLESS] = '1';
logLevel = log.getLevel();
log.setLevel(log.LEVELS.ERROR);
+ process.env.APIFY_LOCAL_STORAGE_DIR = LOCAL_STORAGE_DIR;
+ });
+
+ afterEach(() => {
+ emptyLocalStorageSubdir('key_value_stores/default');
});
afterAll(() => {
@@ -90,4 +96,67 @@ describe('PuppeteerCrawler', () => {
expect(failedCalled).toBe(false);
}
});
+
+ test('should use SessionPool', async () => {
+ const requestList = new Apify.RequestList({
+ sources: [
+ { url: 'http://example.com/?q=1' },
+ ],
+ });
+ const handlePageSessions = [];
+ const goToPageSessions = [];
+ const puppeteerCrawler = new Apify.PuppeteerCrawler({
+ requestList,
+ useSessionPool: true,
+ handlePageFunction: ({ session }) => {
+ handlePageSessions.push(session);
+ return Promise.resolve();
+ },
+ gotoFunction: ({ session }) => {
+ goToPageSessions.push(session);
+ return Apify.launchPuppeteer();
+ },
+ });
+
+ await requestList.initialize();
+ await puppeteerCrawler.run();
+
+ expect(puppeteerCrawler.sessionPool.constructor.name).toEqual('SessionPool');
+ expect(handlePageSessions).toHaveLength(1);
+ expect(goToPageSessions).toHaveLength(1);
+ handlePageSessions.forEach(session => expect(session.constructor.name).toEqual('Session'));
+ goToPageSessions.forEach(session => expect(session.constructor.name).toEqual('Session'));
+ });
+
+ test('should persist cookies per session', async () => {
+ const requestList = new Apify.RequestList({
+ sources: [
+ { url: 'http://example.com/?q=1' },
+ { url: 'http://example.com/?q=2' },
+ { url: 'http://example.com/?q=3' },
+ { url: 'http://example.com/?q=4' },
+ ],
+ });
+ const goToPageSessions = [];
+ const loadedCookies = [];
+ const puppeteerCrawler = new Apify.PuppeteerCrawler({
+ requestList,
+ useSessionPool: true,
+ persistCookiesPerSession: true,
+ handlePageFunction: async ({ session, request }) => {
+ loadedCookies.push(session.getCookieString(request.url));
+ return Promise.resolve();
+ },
+ gotoFunction: async ({ session, page, request }) => {
+ await page.setCookie({ name: 'TEST', value: '12321312312', domain: 'example.com', expires: Date.now() + 100000 });
+ goToPageSessions.push(session);
+ return page.goto(request.url);
+ },
+ });
+
+ await requestList.initialize();
+ await puppeteerCrawler.run();
+ expect(loadedCookies).toHaveLength(4);
+ loadedCookies.forEach(cookie => expect(cookie).toEqual('TEST=12321312312'));
+ });
});