Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

fix: treat `CookieJar` write methods as potentially throwing (#3426)

Copilot473886915a4d
diff --git a/packages/core/src/session_pool/session.ts b/packages/core/src/session_pool/session.ts
index 95c5cd737..df3e9e7c8 100644
--- a/packages/core/src/session_pool/session.ts
+++ b/packages/core/src/session_pool/session.ts
@@ -364,7 +364,11 @@ export class Session implements ISession {
* Sets a cookie within this session for the specific URL.
*/
setCookie(rawCookie: string, url: string): void {
- this.cookieJar.setCookieSync(rawCookie, url);
+ try {
+ this.cookieJar.setCookieSync(rawCookie, url);
+ } catch (e) {
+ this.log.warning('Could not set cookie.', { url, error: (e as Error).message });
+ }
}
/**
@@ -384,7 +388,7 @@ export class Session implements ISession {
// if invalid cookies are provided just log the exception. No need to retry the request automatically.
if (errorMessages.length) {
- this.log.debug('Could not set cookies.', { errorMessages });
+ this.log.warning('Could not set cookies.', { errorMessages });
}
}
diff --git a/packages/http-client/package.json b/packages/http-client/package.json
index e7d4c5b4f..f94278cc9 100644
--- a/packages/http-client/package.json
+++ b/packages/http-client/package.json
@@ -47,6 +47,7 @@
"access": "public"
},
"dependencies": {
+ "@apify/log": "^2.5.32",
"@crawlee/types": "4.0.0",
"tough-cookie": "^6.0.0"
}
diff --git a/packages/http-client/src/base-http-client.ts b/packages/http-client/src/base-http-client.ts
index f2831f0af..94f9e2fc8 100644
--- a/packages/http-client/src/base-http-client.ts
+++ b/packages/http-client/src/base-http-client.ts
@@ -1,6 +1,9 @@
import type { BaseHttpClient as BaseHttpClientInterface, SendRequestOptions } from '@crawlee/types';
import { CookieJar } from 'tough-cookie';
+import type { Log } from '@apify/log';
+import defaultLog from '@apify/log';
+
export interface CustomFetchOptions {
proxyUrl?: string;
}
@@ -11,6 +14,12 @@ export interface CustomFetchOptions {
* implement only the low-level network call in `fetch`.
*/
export abstract class BaseHttpClient implements BaseHttpClientInterface {
+ protected log: Log;
+
+ constructor(log?: Log) {
+ this.log = log ?? defaultLog;
+ }
+
/**
* Perform the raw network request and return a single Response without any
* automatic redirect following or special error handling.
@@ -18,10 +27,16 @@ export abstract class BaseHttpClient implements BaseHttpClientInterface {
protected abstract fetch(input: Request, init?: RequestInit & CustomFetchOptions): Promise<Response>;
private async applyCookies(request: Request, cookieJar: CookieJar): Promise<Request> {
- const cookies = (await cookieJar.getCookies(request.url)).map((x) => x.cookieString().trim()).filter(Boolean);
+ try {
+ const cookies = (await cookieJar.getCookies(request.url))
+ .map((x) => x.cookieString().trim())
+ .filter(Boolean);
- if (cookies?.length > 0) {
- request.headers.set('cookie', cookies.join('; '));
+ if (cookies?.length > 0) {
+ request.headers.set('cookie', cookies.join('; '));
+ }
+ } catch (e) {
+ this.log.warning(`Failed to get cookies for URL "${request.url}": ${(e as Error).message}`);
}
return request;
}
@@ -29,7 +44,13 @@ export abstract class BaseHttpClient implements BaseHttpClientInterface {
private async setCookies(response: Response, cookieJar: CookieJar): Promise<void> {
const setCookieHeaders = response.headers.getSetCookie();
- await Promise.all(setCookieHeaders.map((header) => cookieJar.setCookie(header, response.url)));
+ for (const header of setCookieHeaders) {
+ try {
+ await cookieJar.setCookie(header, response.url);
+ } catch (e) {
+ this.log.warning(`Failed to set cookie for URL "${response.url}": ${(e as Error).message}`);
+ }
+ }
}
private resolveRequestContext(options?: SendRequestOptions): {
diff --git a/test/core/session_pool/session.test.ts b/test/core/session_pool/session.test.ts
index 03bb260b9..7cbd32da0 100644
--- a/test/core/session_pool/session.test.ts
+++ b/test/core/session_pool/session.test.ts
@@ -1,4 +1,4 @@
-import { EVENT_SESSION_RETIRED, Session, SessionPool } from '@crawlee/core';
+import { EVENT_SESSION_RETIRED, log, Session, SessionPool } from '@crawlee/core';
import { ResponseWithUrl } from '@crawlee/http-client';
import { entries, sleep } from '@crawlee/utils';
import { CookieJar } from 'tough-cookie';
@@ -207,6 +207,26 @@ describe('Session - testing session behaviour ', () => {
expect(session.getCookieString(url)).toBe('cookie2=your-cookie');
});
+ test('setCookies will log warning (not throw) on invalid cookies', () => {
+ const url = 'https://www.example.com';
+ // domain 'abc.different.domain' does not match the request URL, so tough-cookie rejects it
+ const cookies = [{ name: 'cookie1', value: 'my-cookie', domain: 'abc.different.domain' }];
+
+ const mockedLog = vitest.mockObject(log, {
+ spy: true,
+ });
+
+ session = new Session({ sessionPool, log: mockedLog } as any);
+ session.setCookies(cookies, url);
+ expect(session.getCookieString(url)).toBe('');
+ expect(mockedLog.warning).toHaveBeenCalledOnce();
+ });
+
+ test('setCookie does not throw on malformed raw cookie string', () => {
+ session = new Session({ sessionPool });
+ expect(() => session.setCookie('garbled!!!@#$%nonsense', 'https://www.example.com')).not.toThrow();
+ });
+
test('setCookies works with hostOnly cookies', () => {
const url = 'https://www.example.com';
const cookies = [