Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
added tests for cookies and moved update cookies function from CheerioCrawler to utils.
Petr Patek4a306e130b9c
diff --git a/src/crawlers/cheerio_crawler.js b/src/crawlers/cheerio_crawler.jsindex cf3ac876d..bcdd6142a 100644--- a/src/crawlers/cheerio_crawler.js+++ b/src/crawlers/cheerio_crawler.js@@ -9,7 +9,7 @@ import { checkParamOrThrow } from 'apify-client/build/utils';import contentTypeParser from 'content-type';import { readStreamToString, concatStreamToBuffer } from 'apify-shared/streams_utilities';import BasicCrawler from './basic_crawler';-import { addTimeoutToPromise, parseContentTypeFromResponse, getCookiesFromResponse, getCookieHeader } from '../utils';+import { addTimeoutToPromise, parseContentTypeFromResponse, updateSessionCookies, getCookieHeader } from '../utils';import { getApifyProxyUrl } from '../actor';import { BASIC_CRAWLER_TIMEOUT_MULTIPLIER, STATUS_CODES_BLOCKED } from '../constants';import { requestAsBrowser } from '../utils_request';@@ -380,7 +380,7 @@ class CheerioCrawler {);if (this.persistCookiesPerSession) {- session = this._updateSessionCookies(session, response);+ session = updateSessionCookies(session, response);}request.loadedUrl = response.url;@@ -583,30 +583,6 @@ class CheerioCrawler {});}- /**- * @param session {Session} - `Session` instance to be updated.- * @param response- * @return {Session}- * @private- */- _updateSessionCookies(session, response) {- // update cookies for session- const newCookies = getCookiesFromResponse(response);- const { cookies: oldCookies } = session;-- for (const cookie of newCookies) {- const cookieIndex = oldCookies.find(c => c.name === cookie.name);- if (cookieIndex) {- oldCookies[cookieIndex] = cookie;- } else {- oldCookies.push(cookie);- }- }- session.cookies = oldCookies;-- return session;- }-/*** Handles blocked request* @param session {Session}diff --git a/src/utils.js b/src/utils.jsindex 189327c2c..76c5530e6 100644--- a/src/utils.js+++ b/src/utils.js@@ -670,6 +670,11 @@ export const parseContentTypeFromResponse = (response) => {};};+/**+ *+ * @param response+ * @return {undefined|Cookie[]}+ */export const getCookiesFromResponse = (response) => {const { headers } = response;let cookies;@@ -683,12 +688,42 @@ export const getCookiesFromResponse = (response) => {return cookies;};+/**+ * Gets cookie string for the "Cookie" header+ * @param cookies {[Cookie]}+ * @return {String|undefined}+ */export const getCookieHeader = (cookies) => {if (cookies.length < 1) {return;}return cookies.map(cookie => cookie && cookie.cookieString()).join(';');};++/**+ * Updates cookies from response to the `Session` instance.+ * @param session {Session} - `Session` instance to be updated.+ * @param response+ * @return {Session}+ *+ */+export const updateSessionCookies = (session, response) => {+ const newCookies = getCookiesFromResponse(response);+ const { cookies: oldCookies } = session;++ for (const cookie of newCookies) {+ const cookieIndex = oldCookies.findIndex(c => c.key === cookie.key);++ if (cookieIndex >= 0) {+ oldCookies[cookieIndex] = cookie;+ } else {+ oldCookies.push(cookie);+ }+ }+ session.cookies = oldCookies;++ return session;+};/*** A namespace that contains various utilities.*diff --git a/test/crawlers/cheerio_crawler.test.js b/test/crawlers/cheerio_crawler.test.jsindex 8d19bfbc5..79cf8decf 100644--- a/test/crawlers/cheerio_crawler.test.js+++ b/test/crawlers/cheerio_crawler.test.js@@ -873,6 +873,62 @@ describe('CheerioCrawler', () => {expect(e.message).toEqual('Cannot use "options.persistCookiesPerSession" without "options.useSessionPool"');}});- // @TODO: TEST COKIE PERSISTANCE.++ test('should save cookies to session', async () => {+ const cookie = 'SESSID=abcd123';+ const sessions = [];+ const requests = [];+ const crawler = new Apify.CheerioCrawler({+ requestList: await getRequestListForMock({+ headers: { 'set-cookie': cookie, 'content-type': 'text/html' },+ statusCode: 200,+ }),+ useSessionPool: true,+ persistCookiesPerSession: true,+ maxRequestRetries: 1,+ handlePageFunction: async ({ session, request }) => {+ sessions.push(session);+ requests.push(request);+ },++ });++ await crawler.run();+ console.log(requests);+ expect(sessions.length).toBeGreaterThan(1);+ sessions.forEach((session) => {+ expect(session.cookies).toHaveLength(1);+ expect(session.cookies[0].toString()).toEqual(cookie);+ });+ });++ test('should send cookies', async () => {+ const cookie = 'SESSID=abcd123';+ const requests = [];+ const crawler = new Apify.CheerioCrawler({+ requestList: await getRequestListForMock({+ headers: { 'set-cookie': cookie, 'content-type': 'text/html' },+ statusCode: 200,+ }),+ useSessionPool: true,+ persistCookiesPerSession: true,+ sessionPoolOptions: {+ maxPoolSize: 1,+ },+ maxRequestRetries: 1,+ maxConcurrency: 1,+ handlePageFunction: async ({ request }) => {+ requests.push(request);+ },++ });++ await crawler.run();+ requests.forEach((req, i) => {+ if (i >= 1) {+ expect(req.headers.Cookie).toEqual(cookie);+ }+ });+ });});});diff --git a/test/utils.test.js b/test/utils.test.jsindex 444a8db39..e48f9914d 100644--- a/test/utils.test.js+++ b/test/utils.test.js@@ -12,6 +12,9 @@ import { ENV_VARS, LOCAL_ENV_VARS } from 'apify-shared/consts';import { Cookie } from 'tough-cookie';import * as utils from '../build/utils';import Apify from '../build/index';+import { Session } from '../build/session_pool/session';+import { updateSessionCookies } from '../src/utils';+import { SessionPool } from '../src/session_pool/session_pool';describe('utils.newClient()', () => {test('reads environment variables correctly', () => {@@ -1008,3 +1011,30 @@ describe('utils.getCookiesFromResponse & utils.getCookieHeader', () => {expect(parsed).toBeFalsy();});});++describe('utils.updateSessionCookies', () => {+ test('should set and update cookies from "set-cookie" header', () => {+ const headers = {};+ const dummyCookies = ['CSRF=e8b667; Domain=example.com; Secure', 'id=a3fWa; Expires=Wed, 21 Oct 2015 07:28:00 GMT'];+ headers['set-cookie'] = dummyCookies;+ let session = new Session({ sessionPool: new SessionPool() });+ session = updateSessionCookies(session, { headers });++ expect(session.cookies).toHaveLength(2);+ expect(session.cookies[0].toString()).toEqual(dummyCookies[0]);+ expect(session.cookies[1].toString()).toEqual(dummyCookies[1]);++ const newDummyCookies = ['CSRF=123121; Domain=example.cz; Secure', 'id=abcdef; Expires=Wed, 21 Oct 2015 07:28:00 GMT'];+ headers['set-cookie'] = newDummyCookies;+ session = updateSessionCookies(session, { headers });++ expect(session.cookies).toHaveLength(2);+ expect(session.cookies[0].toString()).toEqual(newDummyCookies[0]);+ expect(session.cookies[1].toString()).toEqual(newDummyCookies[1]);++ const newCookie = 'ABCD=1231231213; Domain=example-site.cz; Secure';++ session = updateSessionCookies(session, { headers: { 'set-cookie': newCookie } });+ expect(session.cookies).toHaveLength(3);+ });+});