Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

fix: cookie monster is no more (#1414)

Vlad Frangua361147dd711
diff --git a/package-lock.json b/package-lock.json
index 71ede32ce..959112fc4 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -16197,8 +16197,7 @@
"@crawlee/core": "^3.0.1",
"@crawlee/types": "^3.0.1",
"@crawlee/utils": "^3.0.1",
- "ow": "^0.28.1",
- "tough-cookie": "^4.0.0"
+ "ow": "^0.28.1"
},
"engines": {
"node": ">=16.0.0"
@@ -16434,8 +16433,7 @@
"@crawlee/core": "^3.0.1",
"@crawlee/utils": "^3.0.1",
"jquery": "^3.6.0",
- "ow": "^0.28.1",
- "tough-cookie": "^4.0.0"
+ "ow": "^0.28.1"
},
"engines": {
"node": ">=16.0.0"
@@ -17557,8 +17555,7 @@
"@crawlee/core": "^3.0.1",
"@crawlee/types": "^3.0.1",
"@crawlee/utils": "^3.0.1",
- "ow": "^0.28.1",
- "tough-cookie": "^4.0.0"
+ "ow": "^0.28.1"
}
},
"@crawlee/browser-pool": {
@@ -17709,8 +17706,7 @@
"@crawlee/core": "^3.0.1",
"@crawlee/utils": "^3.0.1",
"jquery": "^3.6.0",
- "ow": "^0.28.1",
- "tough-cookie": "^4.0.0"
+ "ow": "^0.28.1"
}
},
"@crawlee/puppeteer": {
diff --git a/packages/basic-crawler/src/internals/basic-crawler.ts b/packages/basic-crawler/src/internals/basic-crawler.ts
index 278018740..1e99b1ad5 100644
--- a/packages/basic-crawler/src/internals/basic-crawler.ts
+++ b/packages/basic-crawler/src/internals/basic-crawler.ts
@@ -17,6 +17,7 @@ import type {
SessionPoolOptions,
} from '@crawlee/core';
import {
+ mergeCookies,
AutoscaledPool,
Configuration,
type CrawlingContext,
@@ -1062,7 +1063,12 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
}
protected _getCookieHeaderFromRequest(request: Request) {
- return request.headers?.Cookie ?? request.headers?.cookie ?? '';
+ if (request.headers?.Cookie && request.headers?.cookie) {
+ this.log.warning(`Encountered mixed casing for the cookie headers for request ${request.url} (${request.id}). Their values will be merged.`);
+ return mergeCookies(request.url, [request.headers.cookie, request.headers.Cookie]);
+ }
+
+ return request.headers?.Cookie || request.headers?.cookie || '';
}
}
diff --git a/packages/browser-crawler/package.json b/packages/browser-crawler/package.json
index 89874371c..0046516a2 100644
--- a/packages/browser-crawler/package.json
+++ b/packages/browser-crawler/package.json
@@ -59,7 +59,6 @@
"@crawlee/core": "^3.0.1",
"@crawlee/types": "^3.0.1",
"@crawlee/utils": "^3.0.1",
- "ow": "^0.28.1",
- "tough-cookie": "^4.0.0"
+ "ow": "^0.28.1"
}
}
diff --git a/packages/browser-crawler/src/internals/browser-crawler.ts b/packages/browser-crawler/src/internals/browser-crawler.ts
index 9848f973a..dfb7b4393 100644
--- a/packages/browser-crawler/src/internals/browser-crawler.ts
+++ b/packages/browser-crawler/src/internals/browser-crawler.ts
@@ -8,6 +8,7 @@ import type {
Session,
} from '@crawlee/core';
import {
+ cookieStringToToughCookie,
enqueueLinks,
EVENT_SESSION_RETIRED,
handleRequestTimeout,
@@ -29,7 +30,6 @@ import type {
import { BROWSER_CONTROLLER_EVENTS, BrowserPool } from '@crawlee/browser-pool';
import type { GotOptionsInit, Response as GotResponse } from 'got-scraping';
import ow from 'ow';
-import { Cookie } from 'tough-cookie';
import type { BatchAddRequestsResult, Cookie as CookieObject } from '@crawlee/types';
import type { BrowserLaunchContext } from './browser-launcher';
@@ -544,8 +544,8 @@ export abstract class BrowserCrawler<
protected async _applyCookies({ session, request, page, browserController }: Context, preHooksCookies: string, postHooksCookies: string) {
const sessionCookie = session?.getCookies(request.url) ?? [];
- const parsedPreHooksCookies = preHooksCookies.split(/ *; */).map((c) => Cookie.parse(c)?.toJSON());
- const parsedPostHooksCookies = postHooksCookies.split(/ *; */).map((c) => Cookie.parse(c)?.toJSON());
+ const parsedPreHooksCookies = preHooksCookies.split(/ *; */).map((c) => cookieStringToToughCookie(c));
+ const parsedPostHooksCookies = postHooksCookies.split(/ *; */).map((c) => cookieStringToToughCookie(c));
await browserController.setCookies(
page,
@@ -553,7 +553,9 @@ export abstract class BrowserCrawler<
...sessionCookie,
...parsedPreHooksCookies,
...parsedPostHooksCookies,
- ].filter((c): c is CookieObject => typeof c !== 'undefined'),
+ ]
+ .filter((c): c is CookieObject => typeof c !== 'undefined' && c !== null)
+ .map((c) => ({ ...c, url: c.domain ? undefined : request.url })),
);
}
diff --git a/packages/cheerio-crawler/src/internals/cheerio-crawler.ts b/packages/cheerio-crawler/src/internals/cheerio-crawler.ts
index f10f078a6..ae838b1a9 100644
--- a/packages/cheerio-crawler/src/internals/cheerio-crawler.ts
+++ b/packages/cheerio-crawler/src/internals/cheerio-crawler.ts
@@ -611,11 +611,52 @@ export class CheerioCrawler extends BasicCrawler<CheerioCrawlingContext> {
*/
private _applyCookies({ session, request }: CrawlingContext, gotOptions: OptionsInit, preHookCookies: string, postHookCookies: string) {
const sessionCookie = session?.getCookieString(request.url) ?? '';
- const alteredGotOptionsCookies = (gotOptions.headers?.Cookie ?? gotOptions.headers?.cookie ?? '') as string;
+ let alteredGotOptionsCookies = (gotOptions.headers?.Cookie || gotOptions.headers?.cookie || '');
- const mergedCookie = mergeCookies(request.url, [sessionCookie, preHookCookies, alteredGotOptionsCookies, postHookCookies]);
+ if (gotOptions.headers?.Cookie && gotOptions.headers?.cookie) {
+ const {
+ Cookie: upperCaseHeader,
+ cookie: lowerCaseHeader,
+ } = gotOptions.headers;
+
+ // eslint-disable-next-line max-len
+ this.log.warning(`Encountered mixed casing for the cookie headers in the got options for request ${request.url} (${request.id}). Their values will be merged`);
+
+ const sourceCookies = [];
+
+ if (Array.isArray(lowerCaseHeader)) {
+ sourceCookies.push(...lowerCaseHeader);
+ } else {
+ sourceCookies.push(lowerCaseHeader);
+ }
+
+ if (Array.isArray(upperCaseHeader)) {
+ sourceCookies.push(...upperCaseHeader);
+ } else {
+ sourceCookies.push(upperCaseHeader);
+ }
+
+ alteredGotOptionsCookies = mergeCookies(request.url, sourceCookies);
+ }
+
+ const sourceCookies = [
+ sessionCookie,
+ preHookCookies,
+ ];
+
+ if (Array.isArray(alteredGotOptionsCookies)) {
+ sourceCookies.push(...alteredGotOptionsCookies);
+ } else {
+ sourceCookies.push(alteredGotOptionsCookies);
+ }
+
+ sourceCookies.push(postHookCookies);
+
+ const mergedCookie = mergeCookies(request.url, sourceCookies);
gotOptions.headers ??= {};
+ Reflect.deleteProperty(gotOptions.headers, 'Cookie');
+ Reflect.deleteProperty(gotOptions.headers, 'cookie');
gotOptions.headers.Cookie = mergedCookie;
}
@@ -690,6 +731,9 @@ export class CheerioCrawler extends BasicCrawler<CheerioCrawlingContext> {
isStream: true,
};
+ // Delete any possible lowercased header for cookie as they are merged in _applyCookies under the uppercase Cookie header
+ Reflect.deleteProperty(requestOptions.headers!, 'cookie');
+
// TODO this is incorrect, the check for man in the middle needs to be done
// on individual proxy level, not on the `proxyConfiguration` level,
// because users can use normal + MITM proxies in a single configuration.
diff --git a/packages/core/src/cookie_utils.ts b/packages/core/src/cookie_utils.ts
new file mode 100644
index 000000000..de0612ca8
--- /dev/null
+++ b/packages/core/src/cookie_utils.ts
@@ -0,0 +1,124 @@
+import type { BrowserLikeResponse, Dictionary, Cookie as CookieObject } from '@crawlee/types';
+import type { IncomingMessage } from 'node:http';
+import { Cookie, CookieJar } from 'tough-cookie';
+import { CookieParseError } from './session_pool/errors';
+import { log } from './log';
+
+/**
+ * @internal
+ */
+export function getCookiesFromResponse(response: IncomingMessage | BrowserLikeResponse | { headers: Dictionary<string | string[]> }): Cookie[] {
+ const headers = typeof response.headers === 'function' ? response.headers() : response.headers;
+ const cookieHeader = headers['set-cookie'] || '';
+
+ try {
+ return Array.isArray(cookieHeader)
+ ? cookieHeader.map((cookie) => Cookie.parse(cookie)!)
+ : [Cookie.parse(cookieHeader)!];
+ } catch (e) {
+ throw new CookieParseError(cookieHeader);
+ }
+}
+
+/**
+ * Calculate cookie expiration date
+ * @param maxAgeSecs
+ * @returns Calculated date by session max age seconds.
+ * @internal
+ */
+export function getDefaultCookieExpirationDate(maxAgeSecs: number) {
+ return new Date(Date.now() + (maxAgeSecs * 1000));
+}
+
+/**
+ * Transforms tough-cookie to puppeteer cookie.
+ * @param toughCookie Cookie from CookieJar
+ * @return Cookie compatible with browser pool
+ * @internal
+ */
+export function toughCookieToBrowserPoolCookie(toughCookie: Cookie): CookieObject {
+ return {
+ name: toughCookie.key,
+ value: toughCookie.value,
+ // Puppeteer and Playwright expect 'expires' to be 'Unix time in seconds', not ms
+ // If there is no expires date (so defaults to Infinity), we don't provide it to the browsers
+ expires: toughCookie.expires === 'Infinity' ? undefined : new Date(toughCookie.expires).getTime() / 1000,
+ domain: toughCookie.domain ?? undefined,
+ path: toughCookie.path ?? undefined,
+ secure: toughCookie.secure,
+ httpOnly: toughCookie.httpOnly,
+ };
+}
+
+/**
+ * Transforms browser-pool cookie to tough-cookie.
+ * @param cookieObject Cookie object (for instance from the `page.cookies` method).
+ * @internal
+ */
+export function browserPoolCookieToToughCookie(cookieObject: CookieObject, maxAgeSecs: number) {
+ const isExpiresValid = cookieObject.expires && typeof cookieObject.expires === 'number' && cookieObject.expires > 0;
+ const expires = isExpiresValid ? new Date(cookieObject.expires! * 1000) : getDefaultCookieExpirationDate(maxAgeSecs);
+ const domain = typeof cookieObject.domain === 'string' && cookieObject.domain.startsWith('.')
+ ? cookieObject.domain.slice(1)
+ : cookieObject.domain;
+
+ return new Cookie({
+ key: cookieObject.name,
+ value: cookieObject.value,
+ expires,
+ domain,
+ path: cookieObject.path,
+ secure: cookieObject.secure,
+ httpOnly: cookieObject.httpOnly,
+ });
+}
+
+/**
+ * @internal
+ * @param cookieString The cookie string to attempt parsing
+ * @returns Browser pool compatible cookie, or null if cookie cannot be parsed
+ */
+export function cookieStringToToughCookie(cookieString: string) {
+ const parsed = Cookie.parse(cookieString);
+
+ if (parsed) {
+ return toughCookieToBrowserPoolCookie(parsed);
+ }
+
+ return null;
+}
+
+/**
+ * Merges multiple cookie strings. Keys are compared case-sensitively, warning will be logged
+ * if we see two cookies with same keys but different casing.
+ * @internal
+ */
+export function mergeCookies(url: string, sourceCookies: string[]): string {
+ const jar = new CookieJar();
+
+ // ignore empty cookies
+ for (const sourceCookieString of sourceCookies) {
+ // ignore empty cookies
+ if (!sourceCookieString) continue;
+
+ const cookies = sourceCookieString.split(/ *; */);
+
+ for (const cookieString of cookies) {
+ // ignore extra spaces
+ if (!cookieString) continue;
+
+ const cookie = Cookie.parse(cookieString)!;
+ const similarKeyCookie = jar.getCookiesSync(url).find((c) => {
+ return cookie.key !== c.key && cookie.key.toLowerCase() === c.key.toLowerCase();
+ });
+
+ if (similarKeyCookie) {
+ log.deprecated(`Found cookies with similar name during cookie merging: '${cookie.key}' and '${similarKeyCookie.key}'`);
+ }
+
+ jar.setCookieSync(cookie, url);
+ }
+ }
+
+ return jar.getCookieStringSync(url);
+}
diff --git a/packages/core/src/crawlers/crawler_utils.ts b/packages/core/src/crawlers/crawler_utils.ts
index c1ae70fe3..779cedcf1 100644
--- a/packages/core/src/crawlers/crawler_utils.ts
+++ b/packages/core/src/crawlers/crawler_utils.ts
@@ -1,6 +1,4 @@
import { TimeoutError } from '@apify/timeout';
-import { Cookie, CookieJar } from 'tough-cookie';
-import { log } from '../log';
import type { Session } from '../session_pool/session';
/**
@@ -13,56 +11,3 @@ export function handleRequestTimeout({ session, errorMessage }: { session?: Sess
const timeoutSecs = Number(timeoutMillis) / 1000;
throw new TimeoutError(`Navigation timed out after ${timeoutSecs} seconds.`);
}
-
-/**
- * Merges multiple cookie strings. Keys are compared case-sensitively, warning will be logged
- * if we see two cookies with same keys but different casing.
- * @internal
- */
-export function mergeCookies(url: string, sourceCookies: string[]): string {
- const jar = new CookieJar();
-
- // ignore empty cookies
- for (const sourceCookieString of sourceCookies.filter((c) => c)) {
- const cookies = sourceCookieString.split(/ *; */).filter((c) => c); // ignore extra spaces
-
- for (const cookieString of cookies) {
- const cookie = Cookie.parse(cookieString)!;
- const similarKeyCookie = jar.getCookiesSync(url).find((c) => {
- return cookie.key !== c.key && cookie.key.toLowerCase() === c.key.toLowerCase();
- });
-
- if (similarKeyCookie) {
- log.deprecated(`Found cookies with similar name during cookie merging: '${cookie.key}' and '${similarKeyCookie.key}'`);
- }
-
- jar.setCookieSync(cookie, url);
- }
- }
-
- return jar.getCookieStringSync(url);
-}
-
-/**
- * @internal
- */
-export function diffCookies(url: string, cookieString1 = '', cookieString2 = ''): string {
- if (cookieString1 === cookieString2 || !cookieString2) {
- return '';
- }
-
- if (!cookieString1) {
- return cookieString2;
- }
-
- const cookies1 = cookieString1.split(/ *; */).filter((item) => Boolean(item)).map((cookie) => Cookie.parse(cookie)!);
- const cookies2 = cookieString2.split(/ *; */).filter((item) => Boolean(item)).map((cookie) => Cookie.parse(cookie)!);
-
- const added = cookies2.filter((newCookie) => {
- return !cookies1.find((oldCookie) => newCookie.toString() === oldCookie.toString());
- });
- const jar = new CookieJar();
- added.forEach((cookie) => jar.setCookieSync(cookie, url));
-
- return jar.getCookieStringSync(url);
-}
diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts
index e7a8a7a1f..a493e2fa8 100644
--- a/packages/core/src/index.ts
+++ b/packages/core/src/index.ts
@@ -13,5 +13,6 @@ export * from './serialization';
export * from './session_pool';
export * from './storages';
export * from './validators';
+export * from './cookie_utils';
export { PseudoUrl } from '@apify/pseudo_url';
export { Dictionary, Awaitable, Constructor, StorageClient } from '@crawlee/types';
diff --git a/packages/core/src/session_pool/index.ts b/packages/core/src/session_pool/index.ts
index 4f266661a..5d387f855 100644
--- a/packages/core/src/session_pool/index.ts
+++ b/packages/core/src/session_pool/index.ts
@@ -2,4 +2,3 @@ export * from './errors';
export * from './events';
export * from './session';
export * from './session_pool';
-export * from './session_utils';
diff --git a/packages/core/src/session_pool/session.ts b/packages/core/src/session_pool/session.ts
index 8fdd0a923..7b4c8c273 100644
--- a/packages/core/src/session_pool/session.ts
+++ b/packages/core/src/session_pool/session.ts
@@ -4,11 +4,12 @@ import type { Dictionary, Cookie as CookieObject, BrowserLikeResponse } from '@c
import type { IncomingMessage } from 'node:http';
import { EventEmitter } from 'node:events';
import ow from 'ow';
-import { Cookie, CookieJar } from 'tough-cookie';
+import type { Cookie } from 'tough-cookie';
+import { CookieJar } from 'tough-cookie';
import { STATUS_CODES_BLOCKED } from '../constants';
import { log as defaultLog } from '../log';
import { EVENT_SESSION_RETIRED } from './events';
-import { getCookiesFromResponse } from './session_utils';
+import { browserPoolCookieToToughCookie, getCookiesFromResponse, getDefaultCookieExpirationDate, toughCookieToBrowserPoolCookie } from '../cookie_utils';
// CONSTANTS
const DEFAULT_SESSION_MAX_AGE_SECS = 3000;
@@ -144,7 +145,7 @@ export class Session {
log = defaultLog,
} = options;
- const { expiresAt = this._getDefaultCookieExpirationDate(maxAgeSecs) } = options;
+ const { expiresAt = getDefaultCookieExpirationDate(maxAgeSecs) } = options;
this.log = log.child({ prefix: 'Session' });
@@ -162,8 +163,6 @@ export class Session {
this.errorScore = errorScore; // indicates number of markBaded request with the session
this.maxUsageCount = maxUsageCount;
this.sessionPool = sessionPool;
-
- this._cookieObjectToTough = this._cookieObjectToTough.bind(this);
}
/**
@@ -308,17 +307,17 @@ export class Session {
* ```
*/
setCookies(cookies: CookieObject[], url: string) {
- const normalizedCookies = cookies.map(this._cookieObjectToTough);
+ const normalizedCookies = cookies.map((c) => browserPoolCookieToToughCookie(c, this.maxAgeSecs));
this._setCookies(normalizedCookies, url);
}
/**
- * Returns cookies in a format compatible with puppeteer and ready to be used with `page.setCookie`.
+ * Returns cookies in a format compatible with puppeteer/playwright and ready to be used with `page.setCookie`.
* @param url website url. Only cookies stored for this url will be returned
*/
getCookies(url: string): CookieObject[] {
const cookies = this.cookieJar.getCookiesSync(url);
- return cookies.map(this._toughCookieToPuppeteer);
+ return cookies.map((c) => toughCookieToBrowserPoolCookie(c));
}
/**
@@ -338,46 +337,6 @@ export class Session {
this.cookieJar.setCookieSync(rawCookie, url);
}
- /**
- * Transforms puppeteer cookie to tough-cookie.
- * @param cookieObject Cookie object (for instance from the `page.cookies` method).
- */
- protected _cookieObjectToTough(cookieObject: CookieObject): Cookie {
- const isExpiresValid = cookieObject.expires && typeof cookieObject.expires === 'number' && cookieObject.expires > 0;
- const expires = isExpiresValid ? new Date(cookieObject.expires! * 1000) : this._getDefaultCookieExpirationDate(this.maxAgeSecs);
- const domain = typeof cookieObject.domain === 'string' && cookieObject.domain.startsWith('.')
- ? cookieObject.domain.slice(1)
- : cookieObject.domain;
-
- return new Cookie({
- key: cookieObject.name,
- value: cookieObject.value,
- expires,
- domain,
- path: cookieObject.path,
- secure: cookieObject.secure,
- httpOnly: cookieObject.httpOnly,
- });
- }
-
- /**
- * Transforms tough-cookie to puppeteer cookie.
- * @param toughCookie Cookie from CookieJar
- * @return Cookie from Puppeteer
- */
- protected _toughCookieToPuppeteer(toughCookie: Cookie): CookieObject {
- return {
- name: toughCookie.key,
- value: toughCookie.value,
- // Puppeteer and Playwright expect 'expires' to be 'Unix time in seconds', not ms
- expires: new Date(toughCookie.expires).getTime() / 1000,
- domain: toughCookie.domain,
- path: toughCookie.path,
- secure: toughCookie.secure,
- httpOnly: toughCookie.httpOnly,
- } as CookieObject;
- }
-
/**
* Sets cookies.
*/
@@ -399,15 +358,6 @@ export class Session {
}
}
- /**
- * Calculate cookie expiration date
- * @param maxAgeSecs
- * @returns Calculated date by session max age seconds.
- */
- protected _getDefaultCookieExpirationDate(maxAgeSecs: number): Date {
- return new Date(Date.now() + (maxAgeSecs * 1000));
- }
-
/**
* Checks if session is not usable. if it is not retires the session.
*/
diff --git a/packages/core/src/session_pool/session_utils.ts b/packages/core/src/session_pool/session_utils.ts
deleted file mode 100644
index bb9b33b1d..000000000
--- a/packages/core/src/session_pool/session_utils.ts
+++ /dev/null
@@ -1,20 +0,0 @@
-import type { BrowserLikeResponse, Dictionary } from '@crawlee/types';
-import type { IncomingMessage } from 'node:http';
-import { Cookie } from 'tough-cookie';
-import { CookieParseError } from './errors';
-
-/**
- * @internal
- */
-export function getCookiesFromResponse(response: IncomingMessage | BrowserLikeResponse | { headers: Dictionary<string | string[]> }): Cookie[] {
- const headers = typeof response.headers === 'function' ? response.headers() : response.headers;
- const cookieHeader = headers['set-cookie'] || '';
-
- try {
- return Array.isArray(cookieHeader)
- ? cookieHeader.map((cookie) => Cookie.parse(cookie)!)
- : [Cookie.parse(cookieHeader)!];
- } catch (e) {
- throw new CookieParseError(cookieHeader);
- }
-}
diff --git a/packages/playwright-crawler/package.json b/packages/playwright-crawler/package.json
index 1b5049c5f..b3da80af6 100644
--- a/packages/playwright-crawler/package.json
+++ b/packages/playwright-crawler/package.json
@@ -60,8 +60,7 @@
"@crawlee/core": "^3.0.1",
"@crawlee/utils": "^3.0.1",
"jquery": "^3.6.0",
- "ow": "^0.28.1",
- "tough-cookie": "^4.0.0"
+ "ow": "^0.28.1"
},
"peerDependencies": {
"playwright": "^1.21.1"
diff --git a/packages/playwright-crawler/src/internals/playwright-crawler.ts b/packages/playwright-crawler/src/internals/playwright-crawler.ts
index 77fb796ac..1764f123c 100644
--- a/packages/playwright-crawler/src/internals/playwright-crawler.ts
+++ b/packages/playwright-crawler/src/internals/playwright-crawler.ts
@@ -4,7 +4,6 @@ import type { BrowserPoolOptions, PlaywrightController, PlaywrightPlugin } from
import type { BrowserCrawlerOptions, BrowserCrawlingContext, BrowserRequestHandler, BrowserHook } from '@crawlee/browser';
import { BrowserCrawler, Configuration, Router } from '@crawlee/browser';
import type { Dictionary } from '@crawlee/types';
-import { Cookie } from 'tough-cookie';
import type { PlaywrightLaunchContext } from './playwright-launcher';
import { PlaywrightLauncher } from './playwright-launcher';
import type { DirectNavigationOptions, PlaywrightContextUtils } from './utils/playwright-utils';
@@ -216,20 +215,6 @@ export class PlaywrightCrawler extends BrowserCrawler<{ browserPlugins: [Playwri
protected override async _navigationHandler(crawlingContext: PlaywrightCrawlingContext, gotoOptions: DirectNavigationOptions) {
return gotoExtended(crawlingContext.page, crawlingContext.request, gotoOptions);
}
-
- protected override async _applyCookies({ session, request, page }: PlaywrightCrawlingContext, preHooksCookies: string, postHooksCookies: string) {
- const sessionCookie = session?.getCookies(request.url) ?? [];
- const parsedPreHooksCookies = preHooksCookies.split(/ *; */).map((c) => Cookie.parse(c)?.toJSON() as PlaywrightCookie | undefined);
- const parsedPostHooksCookies = postHooksCookies.split(/ *; */).map((c) => Cookie.parse(c)?.toJSON() as PlaywrightCookie | undefined);
-
- await page.context().addCookies(
- [
- ...sessionCookie,
- ...parsedPreHooksCookies,
- ...parsedPostHooksCookies,
- ].filter((c): c is PlaywrightCookie => c !== undefined),
- );
- }
}
/**
diff --git a/test/core/crawlers/crawler_utils.test.ts b/test/core/crawlers/crawler_utils.test.ts
deleted file mode 100644
index 79fa5cadd..000000000
--- a/test/core/crawlers/crawler_utils.test.ts
+++ /dev/null
@@ -1,28 +0,0 @@
-import { diffCookies } from '@crawlee/core';
-
-const cookie1 = 'test_cookie=1';
-const cookie2 = 'new_test_cookie=2';
-
-const mergedCookies = [cookie1, cookie2].join(';');
-
-const url = 'https://example.com';
-
-describe('CrawlerUtils', () => {
- describe('diffCookies', () => {
- test('when no cookies are provided for cookiesString2, return an empty string', () => {
- expect(diffCookies(url, cookie1, '')).toEqual('');
- });
-
- test('when no cookies are provided for cookiesString1, return cookiesString2', () => {
- expect(diffCookies(url, '', cookie1)).toEqual(cookie1);
- });
-
- test('when new cookies are provided for cookiesString2, return them', () => {
- expect(diffCookies(url, cookie1, cookie2)).toEqual(cookie2);
- });
-
- test('when new cookies include previous cookies for cookiesString2, return only the new ones', () => {
- expect(diffCookies(url, cookie1, mergedCookies)).toEqual(cookie2);
- });
- });
-});
diff --git a/test/e2e/cheerio-initial-cookies/actor/main.js b/test/e2e/cheerio-initial-cookies/actor/main.js
index 699b87cd3..47aedf7bf 100644
--- a/test/e2e/cheerio-initial-cookies/actor/main.js
+++ b/test/e2e/cheerio-initial-cookies/actor/main.js
@@ -2,18 +2,26 @@ import { Actor } from 'apify';
import { CheerioCrawler, Dataset } from '@crawlee/cheerio';
import { ApifyStorageLocal } from '@apify/storage-local';
-const initialCookies = [
+const expectedCookies = [
{
- name: 'test',
- value: 'testing cookies',
+ name: 'initial_request',
+ value: 'true',
},
{
- name: 'store',
- value: 'value store',
+ name: 'session',
+ value: 'true',
},
{
- name: 'market_place',
- value: 'value market place',
+ name: 'hook_request',
+ value: 'true',
+ },
+ {
+ name: 'got_options_upper_case',
+ value: 'true',
+ },
+ {
+ name: 'got_options_lower_case',
+ value: 'true',
},
];
@@ -25,11 +33,21 @@ const mainOptions = {
await Actor.main(async () => {
const crawler = new CheerioCrawler({
additionalMimeTypes: ['application/json'],
- preNavigationHooks: [({ session, request }) => {
- session.setCookies(initialCookies, request.url);
+ preNavigationHooks: [({ session, request }, gotOptions) => {
+ session.setCookies([
+ {
+ name: 'session',
+ value: 'true',
+ },
+ ], request.url);
+ request.headers.cookie = 'hook_request=true';
+
+ gotOptions.headers ??= {};
+ gotOptions.headers.Cookie = 'got_options_upper_case=true';
+ gotOptions.headers.cookie = 'got_options_lower_case=true';
}],
async requestHandler({ json }) {
- const initialCookiesLength = initialCookies.length;
+ const initialCookiesLength = expectedCookies.length;
const cookieString = json.headers.cookie;
const pageCookies = cookieString.split(';').map((cookie) => {
@@ -38,7 +56,7 @@ await Actor.main(async () => {
});
let numberOfMatchingCookies = 0;
- for (const cookie of initialCookies) {
+ for (const cookie of expectedCookies) {
if (pageCookies.some((pageCookie) => pageCookie.name === cookie.name && pageCookie.value === cookie.value)) {
numberOfMatchingCookies++;
}
@@ -48,5 +66,12 @@ await Actor.main(async () => {
},
});
- await crawler.run(['https://api.apify.com/v2/browser-info']);
+ await crawler.run([
+ {
+ url: 'https://api.apify.com/v2/browser-info',
+ headers: {
+ Cookie: 'initial_request=true',
+ },
+ },
+ ]);
}, mainOptions);
diff --git a/test/e2e/cheerio-initial-cookies/test.mjs b/test/e2e/cheerio-initial-cookies/test.mjs
index b22a8dded..6987142a0 100644
--- a/test/e2e/cheerio-initial-cookies/test.mjs
+++ b/test/e2e/cheerio-initial-cookies/test.mjs
@@ -6,7 +6,7 @@ await initialize(testActorDirname);
const { stats, datasetItems } = await runActor(testActorDirname);
await expect(stats.requestsFinished === 1, 'All requests finished');
-await expect(datasetItems[0].numberOfMatchingCookies === 3, 'Number of page cookies');
+await expect(datasetItems[0].numberOfMatchingCookies === 5, 'Number of page cookies');
await expect(
datasetItems[0].numberOfMatchingCookies === datasetItems[0].initialCookiesLength,
`Page cookies match the initial defined cookies. Number of non-matching cookies is `
diff --git a/test/e2e/playwright-initial-cookies/actor/.gitignore b/test/e2e/playwright-initial-cookies/actor/.gitignore
new file mode 100644
index 000000000..ced7cbfc5
--- /dev/null
+++ b/test/e2e/playwright-initial-cookies/actor/.gitignore
@@ -0,0 +1,7 @@
+.idea
+.DS_Store
+node_modules
+package-lock.json
+apify_storage
+crawlee_storage
+storage
diff --git a/test/e2e/playwright-initial-cookies/actor/Dockerfile b/test/e2e/playwright-initial-cookies/actor/Dockerfile
new file mode 100644
index 000000000..2a1510bd0
--- /dev/null
+++ b/test/e2e/playwright-initial-cookies/actor/Dockerfile
@@ -0,0 +1,22 @@
+FROM node:16 AS builder
+
+COPY /packages ./packages
+COPY /package*.json ./
+RUN npm --quiet set progress=false \
+ && npm install --only=prod --no-optional
+
+FROM apify/actor-node-playwright-chrome:16
+
+RUN rm -r node_modules
+COPY --from=builder /node_modules ./node_modules
+COPY --from=builder /packages ./packages
+COPY --from=builder /package*.json ./
+COPY /apify.json ./
+COPY /main.js ./
+
+RUN echo "Installed NPM packages:" \
+ && (npm list --only=prod --no-optional --all || true) \
+ && echo "Node.js version:" \
+ && node --version \
+ && echo "NPM version:" \
+ && npm --version
diff --git a/test/e2e/playwright-initial-cookies/actor/apify.json b/test/e2e/playwright-initial-cookies/actor/apify.json
new file mode 100644
index 000000000..ca72b5fff
--- /dev/null
+++ b/test/e2e/playwright-initial-cookies/actor/apify.json
@@ -0,0 +1,6 @@
+{
+ "name": "test-playwright-initial-cookies",
+ "version": "0.0",
+ "buildTag": "latest",
+ "env": null
+}
diff --git a/test/e2e/playwright-initial-cookies/actor/main.js b/test/e2e/playwright-initial-cookies/actor/main.js
new file mode 100644
index 000000000..04248a91a
--- /dev/null
+++ b/test/e2e/playwright-initial-cookies/actor/main.js
@@ -0,0 +1,62 @@
+import { Actor } from 'apify';
+import { Dataset, PlaywrightCrawler } from '@crawlee/playwright';
+import { ApifyStorageLocal } from '@apify/storage-local';
+
+const expectedCookies = [
+ {
+ name: 'initial_request',
+ value: 'true',
+ },
+ {
+ name: 'session',
+ value: 'true',
+ },
+ {
+ name: 'hook_request',
+ value: 'true',
+ },
+];
+
+const mainOptions = {
+ exit: Actor.isAtHome(),
+ storage: process.env.STORAGE_IMPLEMENTATION === 'LOCAL' ? new ApifyStorageLocal() : undefined,
+};
+
+await Actor.main(async () => {
+ const crawler = new PlaywrightCrawler({
+ preNavigationHooks: [({ session, request }, goToOptions) => {
+ session.setCookies([
+ {
+ name: 'session',
+ value: 'true',
+ },
+ ], request.url);
+ request.headers.cookie = 'hook_request=true';
+
+ goToOptions.waitUntil = 'networkidle';
+ }],
+ async requestHandler({ page }) {
+ const initialCookiesLength = expectedCookies.length;
+
+ const pageCookies = await page.context().cookies();
+
+ let numberOfMatchingCookies = 0;
+ for (const cookie of expectedCookies) {
+ if (pageCookies.some((pageCookie) => pageCookie.name === cookie.name && pageCookie.value === cookie.value)) {
+ numberOfMatchingCookies++;
+ }
+ }
+
+ await Dataset.pushData({ initialCookiesLength, numberOfMatchingCookies });
+ },
+ });
+
+ await crawler.run([
+ {
+ url: 'https://api.apify.com/v2/browser-info',
+ headers: {
+ Cookie: 'initial_request=true',
+ },
+ },
+ ]);
+}, mainOptions);
diff --git a/test/e2e/playwright-initial-cookies/actor/package.json b/test/e2e/playwright-initial-cookies/actor/package.json
new file mode 100644
index 000000000..4bdf776e7
--- /dev/null
+++ b/test/e2e/playwright-initial-cookies/actor/package.json
@@ -0,0 +1,21 @@
+{
+ "name": "test-playwright-initial-cookies",
+ "version": "0.0.1",
+ "description": "Playwright Test - Initial Cookies",
+ "dependencies": {
+ "apify": "^3.0.0-beta.75",
+ "@apify/storage-local": "^2.1.0",
+ "@crawlee/basic": "file:./packages/basic-crawler",
+ "@crawlee/browser": "file:./packages/browser-crawler",
+ "@crawlee/browser-pool": "file:./packages/browser-pool",
+ "@crawlee/core": "file:./packages/core",
+ "@crawlee/memory-storage": "file:./packages/memory-storage",
+ "@crawlee/playwright": "file:./packages/playwright-crawler",
+ "@crawlee/types": "file:./packages/types",
+ "@crawlee/utils": "file:./packages/utils",
+ "playwright": "*"
+ },
+ "scripts": { "start": "node main.js" },
+ "type": "module",
+ "license": "ISC"
+}
diff --git a/test/e2e/playwright-initial-cookies/test.mjs b/test/e2e/playwright-initial-cookies/test.mjs
new file mode 100644
index 000000000..a93bcf543
--- /dev/null
+++ b/test/e2e/playwright-initial-cookies/test.mjs
@@ -0,0 +1,14 @@
+import { initialize, getActorTestDir, runActor, expect } from '../tools.mjs';
+
+const testActorDirname = getActorTestDir(import.meta.url);
+await initialize(testActorDirname);
+
+const { stats, datasetItems } = await runActor(testActorDirname, 16384);
+
+await expect(stats.requestsFinished === 1, 'All requests finished');
+await expect(datasetItems[0].numberOfMatchingCookies === 3, 'Number of page cookies');
+await expect(
+ datasetItems[0].numberOfMatchingCookies === datasetItems[0].initialCookiesLength,
+ `Page cookies match the initial defined cookies. Number of non-matching cookies is `
+ + `${datasetItems[0].initialCookiesLength - datasetItems[0].numberOfMatchingCookies}`,
+);
diff --git a/test/e2e/puppeteer-initial-cookies/actor/main.js b/test/e2e/puppeteer-initial-cookies/actor/main.js
index fff341937..81689460b 100644
--- a/test/e2e/puppeteer-initial-cookies/actor/main.js
+++ b/test/e2e/puppeteer-initial-cookies/actor/main.js
@@ -2,18 +2,18 @@ import { Actor } from 'apify';
import { Dataset, PuppeteerCrawler } from '@crawlee/puppeteer';
import { ApifyStorageLocal } from '@apify/storage-local';
-const initialCookies = [
+const expectedCookies = [
{
- name: 'test',
- value: 'testing cookies',
+ name: 'initial_request',
+ value: 'true',
},
{
- name: 'store',
- value: 'value store',
+ name: 'session',
+ value: 'true',
},
{
- name: 'market_place',
- value: 'value market place',
+ name: 'hook_request',
+ value: 'true',
},
];
@@ -25,16 +25,23 @@ const mainOptions = {
await Actor.main(async () => {
const crawler = new PuppeteerCrawler({
preNavigationHooks: [({ session, request }, goToOptions) => {
- session.setCookies(initialCookies, request.url);
+ session.setCookies([
+ {
+ name: 'session',
+ value: 'true',
+ },
+ ], request.url);
+ request.headers.cookie = 'hook_request=true';
+
goToOptions.waitUntil = ['networkidle2'];
}],
async requestHandler({ page }) {
- const initialCookiesLength = initialCookies.length;
+ const initialCookiesLength = expectedCookies.length;
const pageCookies = await page.cookies();
let numberOfMatchingCookies = 0;
- for (const cookie of initialCookies) {
+ for (const cookie of expectedCookies) {
if (pageCookies.some((pageCookie) => pageCookie.name === cookie.name && pageCookie.value === cookie.value)) {
numberOfMatchingCookies++;
}
@@ -44,5 +51,12 @@ await Actor.main(async () => {
},
});
- await crawler.run(['https://api.apify.com/v2/browser-info']);
+ await crawler.run([
+ {
+ url: 'https://api.apify.com/v2/browser-info',
+ headers: {
+ Cookie: 'initial_request=true',
+ },
+ },
+ ]);
}, mainOptions);
diff --git a/test/e2e/run.mjs b/test/e2e/run.mjs
index c2f87204e..d689b63b9 100644
--- a/test/e2e/run.mjs
+++ b/test/e2e/run.mjs
@@ -35,7 +35,7 @@ async function run() {
throw new Error(`Unknown storage provided: '${process.env.STORAGE_IMPLEMENTATION}'`);
}
- execSync(`npm install @apify/storage-local`, { stdio: 'inherit' });
+ execSync(`npm install --no-save @apify/storage-local`, { stdio: 'inherit' });
console.log(`Running E2E tests with storage implementation '${process.env.STORAGE_IMPLEMENTATION}'`);