Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Improved session error handling when setting invalid or empty cookie.

Petr Patekbb366b7950d6
diff --git a/src/session_pool/errors.js b/src/session_pool/errors.js
new file mode 100644
index 000000000..ebee49a4a
--- /dev/null
+++ b/src/session_pool/errors.js
@@ -0,0 +1,8 @@
+export class CookieParseError extends Error {
+ constructor(cookieHeaderString) {
+ super(`Could not parse cookie header string: ${cookieHeaderString}`);
+ this.cookieHeaderString = cookieHeaderString;
+
+ Error.captureStackTrace(this, CookieParseError);
+ }
+}
diff --git a/src/session_pool/session.js b/src/session_pool/session.js
index 207a503d4..00da7d0af 100644
--- a/src/session_pool/session.js
+++ b/src/session_pool/session.js
@@ -1,5 +1,5 @@
import { cryptoRandomObjectId } from 'apify-shared/utilities';
-
+import log from 'apify-shared/log';
import { checkParamOrThrow } from 'apify-client/build/utils';
import tough from 'tough-cookie';
import EVENTS from './events';
@@ -200,9 +200,14 @@ export class Session {
* @param response
*/
putResponse(response) {
- const cookies = getCookiesFromResponse(response).filter(c => c);
+ try {
+ const cookies = getCookiesFromResponse(response).filter(c => c);
- this.setCookies(cookies, response.url);
+ this.setCookies(cookies, response.url);
+ } catch (e) {
+ // if invalid Cookie header is provided just log the exception.
+ log.exception(e, 'Session: Could not get cookies from response');
+ }
}
/**
diff --git a/src/session_pool/session_utils.js b/src/session_pool/session_utils.js
index cb3d40db2..c55386e9c 100644
--- a/src/session_pool/session_utils.js
+++ b/src/session_pool/session_utils.js
@@ -1,4 +1,5 @@
import { Cookie } from 'tough-cookie';
+import { CookieParseError } from './errors';
/**
*
@@ -7,13 +8,11 @@ import { Cookie } from 'tough-cookie';
*/
export const getCookiesFromResponse = (response) => {
const { headers } = response;
- let cookies;
+ const cookieHeader = headers['set-cookie'] || '';
- if (Array.isArray(headers['set-cookie'])) {
- cookies = headers['set-cookie'].map(Cookie.parse);
- } else {
- cookies = [Cookie.parse(headers['set-cookie'])];
+ try {
+ return Array.isArray(cookieHeader) ? cookieHeader.map(Cookie.parse) : [Cookie.parse(cookieHeader)];
+ } catch (e) {
+ throw new CookieParseError(cookieHeader);
}
-
- return cookies;
};
diff --git a/test/session_pool/session.test.js b/test/session_pool/session.test.js
index 15b622e75..d345ce0f4 100644
--- a/test/session_pool/session.test.js
+++ b/test/session_pool/session.test.js
@@ -66,6 +66,17 @@ describe('Session - testing session behaviour ', () => {
expect(session.isBlocked()).toBe(true);
expect(session.isUsable()).toBe(false);
});
+ test('should not throw on invalid Cookie header', () => {
+ let error;
+
+ try {
+ session.putResponse({ headers: { Cookie: 'invaldi*{*{*{*-----***@s' } });
+ } catch (e) {
+ error = e;
+ }
+
+ expect(error).toBeUndefined();
+ });
test('should markGood session', () => {
session.markGood();