Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

fix: merge cookies provided by user with session cookies (#1201)

Martin Adámekcccea783a0d2
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 4dff82812..2affc0927 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -11,6 +11,7 @@
- support channel and user links in YouTube regex (#1178)
- fix: allow passing `requestsFromUrl` to `RequestListOptions` in TS (#1191)
- allow passing `forceCloud` down to the KV store (#1186), closes #752
+- merge cookies from session with user provided ones (#1201), closes #1197
2.0.7 / 2021/09/08
====================
diff --git a/src/crawlers/cheerio_crawler.js b/src/crawlers/cheerio_crawler.js
index 6798820e9..984615417 100644
--- a/src/crawlers/cheerio_crawler.js
+++ b/src/crawlers/cheerio_crawler.js
@@ -11,6 +11,7 @@ import { TimeoutError } from 'got-scraping';
import { BASIC_CRAWLER_TIMEOUT_BUFFER_SECS } from '../constants';
import { addTimeoutToPromise, parseContentTypeFromResponse } from '../utils';
import { requestAsBrowser } from '../utils_request';
+import { diffCookies, mergeCookies } from './crawler_utils';
import { BasicCrawler } from './basic_crawler'; // eslint-disable-line import/no-duplicates
import CrawlerExtension from './crawler_extension';
@@ -481,11 +482,6 @@ class CheerioCrawler extends BasicCrawler {
({ request, response }) => this._abortDownloadOfBody(request, response),
...postNavigationHooks,
];
- /**
- * @type {RequestAsBrowserOptions}
- * @ignore
- * */
- this.defaultRequestAsBrowserOptions = {};
if (this.useSessionPool) {
this.persistCookiesPerSession = persistCookiesPerSession !== undefined ? persistCookiesPerSession : true;
@@ -611,10 +607,17 @@ class CheerioCrawler extends BasicCrawler {
await this.prepareRequestFunction(crawlingContext);
}
- const requestAsBrowserOptions = { ...this.defaultRequestAsBrowserOptions };
- await this._executeHooks(this.preNavigationHooks, crawlingContext, requestAsBrowserOptions);
+ const requestAsBrowserOptions = {};
+
+ if (this.useSessionPool) {
+ this._applySessionCookie(crawlingContext, requestAsBrowserOptions);
+ }
+
const { request, session } = crawlingContext;
+ const cookieSnapshot = request.headers.Cookie ?? request.headers.cookie;
+ await this._executeHooks(this.preNavigationHooks, crawlingContext, requestAsBrowserOptions);
const proxyUrl = crawlingContext.proxyInfo && crawlingContext.proxyInfo.url;
+ this._mergeRequestCookieDiff(request, cookieSnapshot, requestAsBrowserOptions);
crawlingContext.response = await addTimeoutToPromise(
this._requestFunction({ request, session, proxyUrl, requestAsBrowserOptions }),
@@ -630,6 +633,32 @@ class CheerioCrawler extends BasicCrawler {
}
}
+ /**
+ * When users change `request.headers.cookie` inside preNavigationHook, the change would be ignored,
+ * as `request.headers` are already merged into the `requestAsBrowserOptions`. This method is using
+ * old `request.headers` snapshot (before hooks are executed), makes a diff with the cookie value
+ * after hooks are executed, and merges any new cookies back to `requestAsBrowserOptions`.
+ *
+ * This way we can still use both `requestAsBrowserOptions` and `context.request` in the hooks (not both).
+ *
+ * @param {Request} request
+ * @param {string} cookieSnapshot
+ * @param {RequestAsBrowserOptions} requestAsBrowserOptions
+ * @private
+ * @ignore
+ * @internal
+ */
+ _mergeRequestCookieDiff(request, cookieSnapshot, requestAsBrowserOptions) {
+ const cookieDiff = diffCookies(request.url, cookieSnapshot, request.headers.Cookie ?? request.headers.cookie);
+
+ if (cookieDiff.length > 0) {
+ requestAsBrowserOptions.headers.Cookie = mergeCookies(request.url, [
+ requestAsBrowserOptions.headers.Cookie,
+ cookieDiff,
+ ]);
+ }
+ }
+
/**
* Function to make the HTTP request. It performs optimizations
* on the request such as only downloading the request body if the
@@ -646,11 +675,6 @@ class CheerioCrawler extends BasicCrawler {
* @internal
*/
async _requestFunction({ request, session, proxyUrl, requestAsBrowserOptions }) {
- if (this.useSessionPool) {
- const { headers } = request;
- headers.Cookie = session.getCookieString(request.url);
- }
-
const opts = this._getRequestOptions(request, session, proxyUrl, requestAsBrowserOptions);
let responseWithStream;
@@ -667,6 +691,27 @@ class CheerioCrawler extends BasicCrawler {
return responseWithStream;
}
+ /**
+ * Sets the cookie header to `requestAsBrowserOptions` based on provided session and request. If some cookies were already set,
+ * the session cookie will be merged with them. User provided cookies on `request` object have precedence.
+ *
+ * @param {CrawlingContext} crawlingContext
+ * @param {RequestAsBrowserOptions} requestAsBrowserOptions
+ * @return {void}
+ * @ignore
+ * @private
+ * @internal
+ */
+ _applySessionCookie({ request, session }, requestAsBrowserOptions) {
+ const userCookie = request.headers.Cookie ?? request.headers.cookie;
+ const sessionCookie = session.getCookieString(request.url);
+
+ // merge cookies from all possible sources
+ requestAsBrowserOptions.headers = {
+ Cookie: mergeCookies(request.url, [sessionCookie, userCookie]),
+ };
+ }
+
/**
* Encodes and parses response according to the provided content type
* @param {Request} request
diff --git a/src/crawlers/crawler_utils.js b/src/crawlers/crawler_utils.js
index f7ad844b1..5f1fa5d6d 100644
--- a/src/crawlers/crawler_utils.js
+++ b/src/crawlers/crawler_utils.js
@@ -1,6 +1,8 @@
+import { Cookie, CookieJar } from 'tough-cookie';
// TYPE IMPORTS
/* eslint-disable no-unused-vars,import/named,import/no-duplicates,import/order */
import { Session } from '../session_pool/session';
+import log from '../utils_log';
/* eslint-enable no-unused-vars,import/named,import/no-duplicates,import/order */
/**
@@ -29,3 +31,60 @@ export function throwOnBlockedRequest(session, statusCode) {
throw new Error(`Request blocked - received ${statusCode} status code.`);
}
}
+
+/**
+ * Merges multiple cookie strings. Keys are compared case-sensitively, warning will be logged
+ * if we see two cookies with same keys but different casing.
+ *
+ * @param {string} url
+ * @param {string[]} sourceCookies
+ * @return {string}
+ * @private
+ */
+export function mergeCookies(url, sourceCookies) {
+ const jar = new CookieJar();
+
+ // ignore empty cookies
+ for (const sourceCookieString of sourceCookies.filter((c) => c)) {
+ const cookies = sourceCookieString.split(/ *; */); // ignore extra spaces
+
+ for (const cookieString of cookies) {
+ const cookie = Cookie.parse(cookieString);
+ const similarKeyCookie = jar.getCookiesSync(url).find((c) => {
+ return cookie.key !== c.key && cookie.key.toLowerCase() === c.key.toLowerCase();
+ });
+
+ if (similarKeyCookie) {
+ log.deprecated(`Found cookies with similar name during cookie merging: '${cookie.key}' and '${similarKeyCookie.key}'`);
+ }
+
+ jar.setCookieSync(cookie, url);
+ }
+ }
+
+ return jar.getCookieStringSync(url);
+}
+
+/**
+ * @param {string} url
+ * @param {string} cookieString1
+ * @param {string} cookieString2
+ * @return {string}
+ * @private
+ */
+export function diffCookies(url, cookieString1, cookieString2) {
+ if (cookieString1 === cookieString2) {
+ return '';
+ }
+
+ const cookies1 = cookieString1.split(/ *; */).map((cookie) => Cookie.parse(cookie));
+ const cookies2 = cookieString2.split(/ *; */).map((cookie) => Cookie.parse(cookie));
+
+ const added = cookies2.filter((newCookie) => {
+ return !cookies1.find((oldCookie) => newCookie.toString() === oldCookie.toString());
+ });
+ const jar = new CookieJar();
+ added.forEach((cookie) => jar.setCookieSync(cookie, url));
+
+ return jar.getCookieStringSync(url);
+}
diff --git a/test/crawlers/cheerio_crawler.test.js b/test/crawlers/cheerio_crawler.test.js
index 71f61ded0..23f564744 100644
--- a/test/crawlers/cheerio_crawler.test.js
+++ b/test/crawlers/cheerio_crawler.test.js
@@ -8,6 +8,7 @@ import bodyParser from 'body-parser';
import sinon from 'sinon';
import { Readable } from 'stream';
import iconv from 'iconv-lite';
+import { Log } from '@apify/log';
import log from '../../build/utils_log';
import Apify from '../../build';
import { sleep } from '../../build/utils';
@@ -18,6 +19,7 @@ import * as utilsRequest from '../../build/utils_request';
import CrawlerExtension from '../../build/crawlers/crawler_extension';
import Request from '../../build/request';
import AutoscaledPool from '../../build/autoscaling/autoscaled_pool';
+import { mergeCookies } from '../../build/crawlers/crawler_utils';
const HOST = '127.0.0.1';
@@ -96,6 +98,10 @@ app.post('/mock', (req, res) => {
res.status(statusCode).send(body);
});
+app.get('/headers', (req, res) => {
+ res.status(200).json({ headers: req.headers });
+});
+
app.get('/invalidContentType', (req, res) => {
res.send({ some: 'json' });
});
@@ -904,12 +910,113 @@ describe('CheerioCrawler', () => {
});
+ const spy = jest.spyOn(utilsRequest, 'requestAsBrowser');
await crawler.run();
requests.forEach((req, i) => {
if (i >= 1) {
- expect(req.headers.Cookie).toEqual(cookie);
+ expect(spy.mock.calls[i][0].headers.Cookie).toBe(cookie);
}
});
+ spy.mockRestore();
+ });
+
+ test('should merge cookies set in pre-nav hook with the session ones', async () => {
+ const responses = [];
+ const requestAsBrowserOptions = [];
+ const crawler = new Apify.CheerioCrawler({
+ requestList: await Apify.openRequestList(null, [{
+ url: `http://${HOST}:${port}/headers`,
+ headers: { cookie: 'foo=bar2; baz=123' },
+ }]),
+ useSessionPool: true,
+ persistCookiesPerSession: false,
+ sessionPoolOptions: {
+ maxPoolSize: 1,
+ },
+ handlePageFunction: async ({ json }) => {
+ responses.push(json);
+ },
+ preNavigationHooks: [(context, options) => {
+ requestAsBrowserOptions.push(options);
+ }],
+ });
+
+ const sessSpy = jest.spyOn(Session.prototype, 'getCookieString');
+ sessSpy.mockReturnValueOnce('foo=bar1; other=cookie1; coo=kie');
+ await crawler.run();
+ expect(responses).toHaveLength(1);
+ expect(responses[0]).toMatchObject({
+ headers: {
+ cookie: 'foo=bar2; other=cookie1; coo=kie; baz=123',
+ },
+ });
+ expect(requestAsBrowserOptions).toHaveLength(1);
+ expect(requestAsBrowserOptions[0]).toMatchObject({
+ headers: {
+ Cookie: 'foo=bar2; other=cookie1; coo=kie; baz=123', // header name normalized to `Cookie`
+ },
+ });
+ });
+
+ test('should work with cookies adjusted on `context.request` in pre-nav hook', async () => {
+ const responses = [];
+ const crawler = new Apify.CheerioCrawler({
+ requestList: await Apify.openRequestList(null, [{
+ url: `http://${HOST}:${port}/headers`,
+ headers: { cookie: 'foo=bar2; baz=123' },
+ }]),
+ useSessionPool: true,
+ persistCookiesPerSession: false,
+ sessionPoolOptions: {
+ maxPoolSize: 1,
+ },
+ handlePageFunction: async ({ json }) => {
+ responses.push(json);
+ },
+ preNavigationHooks: [({ request }) => {
+ request.headers.Cookie = 'foo=override; coo=kie';
+ }],
+ });
+
+ await crawler.run();
+ expect(responses).toHaveLength(1);
+ expect(responses[0]).toMatchObject({
+ headers: {
+ cookie: 'foo=override; baz=123; coo=kie',
+ },
+ });
+ });
+
+ test('mergeCookies()', async () => {
+ const deprecatedSpy = jest.spyOn(Log.prototype, 'deprecated');
+ const cookie1 = mergeCookies('https://example.com', [
+ 'foo=bar1; other=cookie1 ; coo=kie',
+ 'foo=bar2; baz=123',
+ 'other=cookie2;foo=bar3',
+ ]);
+ expect(cookie1).toBe('foo=bar3; other=cookie2; coo=kie; baz=123');
+ expect(deprecatedSpy).not.toBeCalled();
+
+ const cookie2 = mergeCookies('https://example.com', [
+ 'Foo=bar1; other=cookie1 ; coo=kie',
+ 'foo=bar2; baz=123',
+ 'Other=cookie2;foo=bar3',
+ ]);
+ expect(cookie2).toBe('Foo=bar1; other=cookie1; coo=kie; foo=bar3; baz=123; Other=cookie2');
+ expect(deprecatedSpy).toBeCalledTimes(3);
+ expect(deprecatedSpy).toBeCalledWith(`Found cookies with similar name during cookie merging: 'foo' and 'Foo'`);
+ expect(deprecatedSpy).toBeCalledWith(`Found cookies with similar name during cookie merging: 'Other' and 'other'`);
+ deprecatedSpy.mockClear();
+
+ const cookie3 = mergeCookies('https://example.com', [
+ 'foo=bar1; Other=cookie1 ; Coo=kie',
+ 'foo=bar2; baz=123',
+ 'Other=cookie2;Foo=bar3;coo=kee',
+ ]);
+ expect(cookie3).toBe('foo=bar2; Other=cookie2; Coo=kie; baz=123; Foo=bar3; coo=kee');
+ expect(deprecatedSpy).toBeCalledTimes(2);
+ expect(deprecatedSpy).toBeCalledWith(`Found cookies with similar name during cookie merging: 'Foo' and 'foo'`);
+ expect(deprecatedSpy).toBeCalledWith(`Found cookies with similar name during cookie merging: 'coo' and 'Coo'`);
});
test('should pass session to prepareRequestFunction when Session pool is used', async () => {