Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
fix: cookie monster is no more (#1414)
Vlad Frangua361147dd711
diff --git a/package-lock.json b/package-lock.jsonindex 71ede32ce..959112fc4 100644--- a/package-lock.json+++ b/package-lock.json@@ -16197,8 +16197,7 @@"@crawlee/core": "^3.0.1","@crawlee/types": "^3.0.1","@crawlee/utils": "^3.0.1",- "ow": "^0.28.1",- "tough-cookie": "^4.0.0"+ "ow": "^0.28.1"},"engines": {"node": ">=16.0.0"@@ -16434,8 +16433,7 @@"@crawlee/core": "^3.0.1","@crawlee/utils": "^3.0.1","jquery": "^3.6.0",- "ow": "^0.28.1",- "tough-cookie": "^4.0.0"+ "ow": "^0.28.1"},"engines": {"node": ">=16.0.0"@@ -17557,8 +17555,7 @@"@crawlee/core": "^3.0.1","@crawlee/types": "^3.0.1","@crawlee/utils": "^3.0.1",- "ow": "^0.28.1",- "tough-cookie": "^4.0.0"+ "ow": "^0.28.1"}},"@crawlee/browser-pool": {@@ -17709,8 +17706,7 @@"@crawlee/core": "^3.0.1","@crawlee/utils": "^3.0.1","jquery": "^3.6.0",- "ow": "^0.28.1",- "tough-cookie": "^4.0.0"+ "ow": "^0.28.1"}},"@crawlee/puppeteer": {diff --git a/packages/basic-crawler/src/internals/basic-crawler.ts b/packages/basic-crawler/src/internals/basic-crawler.tsindex 278018740..1e99b1ad5 100644--- a/packages/basic-crawler/src/internals/basic-crawler.ts+++ b/packages/basic-crawler/src/internals/basic-crawler.ts@@ -17,6 +17,7 @@ import type {SessionPoolOptions,} from '@crawlee/core';import {+ mergeCookies,AutoscaledPool,Configuration,type CrawlingContext,@@ -1062,7 +1063,12 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext}protected _getCookieHeaderFromRequest(request: Request) {- return request.headers?.Cookie ?? request.headers?.cookie ?? '';+ if (request.headers?.Cookie && request.headers?.cookie) {+ this.log.warning(`Encountered mixed casing for the cookie headers for request ${request.url} (${request.id}). Their values will be merged.`);+ return mergeCookies(request.url, [request.headers.cookie, request.headers.Cookie]);+ }++ return request.headers?.Cookie || request.headers?.cookie || '';}}diff --git a/packages/browser-crawler/package.json b/packages/browser-crawler/package.jsonindex 89874371c..0046516a2 100644--- a/packages/browser-crawler/package.json+++ b/packages/browser-crawler/package.json@@ -59,7 +59,6 @@"@crawlee/core": "^3.0.1","@crawlee/types": "^3.0.1","@crawlee/utils": "^3.0.1",- "ow": "^0.28.1",- "tough-cookie": "^4.0.0"+ "ow": "^0.28.1"}}diff --git a/packages/browser-crawler/src/internals/browser-crawler.ts b/packages/browser-crawler/src/internals/browser-crawler.tsindex 9848f973a..dfb7b4393 100644--- a/packages/browser-crawler/src/internals/browser-crawler.ts+++ b/packages/browser-crawler/src/internals/browser-crawler.ts@@ -8,6 +8,7 @@ import type {Session,} from '@crawlee/core';import {+ cookieStringToToughCookie,enqueueLinks,EVENT_SESSION_RETIRED,handleRequestTimeout,@@ -29,7 +30,6 @@ import type {import { BROWSER_CONTROLLER_EVENTS, BrowserPool } from '@crawlee/browser-pool';import type { GotOptionsInit, Response as GotResponse } from 'got-scraping';import ow from 'ow';-import { Cookie } from 'tough-cookie';import type { BatchAddRequestsResult, Cookie as CookieObject } from '@crawlee/types';import type { BrowserLaunchContext } from './browser-launcher';@@ -544,8 +544,8 @@ export abstract class BrowserCrawler<protected async _applyCookies({ session, request, page, browserController }: Context, preHooksCookies: string, postHooksCookies: string) {const sessionCookie = session?.getCookies(request.url) ?? [];- const parsedPreHooksCookies = preHooksCookies.split(/ *; */).map((c) => Cookie.parse(c)?.toJSON());- const parsedPostHooksCookies = postHooksCookies.split(/ *; */).map((c) => Cookie.parse(c)?.toJSON());+ const parsedPreHooksCookies = preHooksCookies.split(/ *; */).map((c) => cookieStringToToughCookie(c));+ const parsedPostHooksCookies = postHooksCookies.split(/ *; */).map((c) => cookieStringToToughCookie(c));await browserController.setCookies(page,@@ -553,7 +553,9 @@ export abstract class BrowserCrawler<...sessionCookie,...parsedPreHooksCookies,...parsedPostHooksCookies,- ].filter((c): c is CookieObject => typeof c !== 'undefined'),+ ]+ .filter((c): c is CookieObject => typeof c !== 'undefined' && c !== null)+ .map((c) => ({ ...c, url: c.domain ? undefined : request.url })),);}diff --git a/packages/cheerio-crawler/src/internals/cheerio-crawler.ts b/packages/cheerio-crawler/src/internals/cheerio-crawler.tsindex f10f078a6..ae838b1a9 100644--- a/packages/cheerio-crawler/src/internals/cheerio-crawler.ts+++ b/packages/cheerio-crawler/src/internals/cheerio-crawler.ts@@ -611,11 +611,52 @@ export class CheerioCrawler extends BasicCrawler<CheerioCrawlingContext> {*/private _applyCookies({ session, request }: CrawlingContext, gotOptions: OptionsInit, preHookCookies: string, postHookCookies: string) {const sessionCookie = session?.getCookieString(request.url) ?? '';- const alteredGotOptionsCookies = (gotOptions.headers?.Cookie ?? gotOptions.headers?.cookie ?? '') as string;+ let alteredGotOptionsCookies = (gotOptions.headers?.Cookie || gotOptions.headers?.cookie || '');- const mergedCookie = mergeCookies(request.url, [sessionCookie, preHookCookies, alteredGotOptionsCookies, postHookCookies]);+ if (gotOptions.headers?.Cookie && gotOptions.headers?.cookie) {+ const {+ Cookie: upperCaseHeader,+ cookie: lowerCaseHeader,+ } = gotOptions.headers;++ // eslint-disable-next-line max-len+ this.log.warning(`Encountered mixed casing for the cookie headers in the got options for request ${request.url} (${request.id}). Their values will be merged`);++ const sourceCookies = [];++ if (Array.isArray(lowerCaseHeader)) {+ sourceCookies.push(...lowerCaseHeader);+ } else {+ sourceCookies.push(lowerCaseHeader);+ }++ if (Array.isArray(upperCaseHeader)) {+ sourceCookies.push(...upperCaseHeader);+ } else {+ sourceCookies.push(upperCaseHeader);+ }++ alteredGotOptionsCookies = mergeCookies(request.url, sourceCookies);+ }++ const sourceCookies = [+ sessionCookie,+ preHookCookies,+ ];++ if (Array.isArray(alteredGotOptionsCookies)) {+ sourceCookies.push(...alteredGotOptionsCookies);+ } else {+ sourceCookies.push(alteredGotOptionsCookies);+ }++ sourceCookies.push(postHookCookies);++ const mergedCookie = mergeCookies(request.url, sourceCookies);gotOptions.headers ??= {};+ Reflect.deleteProperty(gotOptions.headers, 'Cookie');+ Reflect.deleteProperty(gotOptions.headers, 'cookie');gotOptions.headers.Cookie = mergedCookie;}@@ -690,6 +731,9 @@ export class CheerioCrawler extends BasicCrawler<CheerioCrawlingContext> {isStream: true,};+ // Delete any possible lowercased header for cookie as they are merged in _applyCookies under the uppercase Cookie header+ Reflect.deleteProperty(requestOptions.headers!, 'cookie');+// TODO this is incorrect, the check for man in the middle needs to be done// on individual proxy level, not on the `proxyConfiguration` level,// because users can use normal + MITM proxies in a single configuration.diff --git a/packages/core/src/cookie_utils.ts b/packages/core/src/cookie_utils.tsnew file mode 100644index 000000000..de0612ca8--- /dev/null+++ b/packages/core/src/cookie_utils.ts@@ -0,0 +1,124 @@+import type { BrowserLikeResponse, Dictionary, Cookie as CookieObject } from '@crawlee/types';+import type { IncomingMessage } from 'node:http';+import { Cookie, CookieJar } from 'tough-cookie';+import { CookieParseError } from './session_pool/errors';+import { log } from './log';++/**+ * @internal+ */+export function getCookiesFromResponse(response: IncomingMessage | BrowserLikeResponse | { headers: Dictionary<string | string[]> }): Cookie[] {+ const headers = typeof response.headers === 'function' ? response.headers() : response.headers;+ const cookieHeader = headers['set-cookie'] || '';++ try {+ return Array.isArray(cookieHeader)+ ? cookieHeader.map((cookie) => Cookie.parse(cookie)!)+ : [Cookie.parse(cookieHeader)!];+ } catch (e) {+ throw new CookieParseError(cookieHeader);+ }+}++/**+ * Calculate cookie expiration date+ * @param maxAgeSecs+ * @returns Calculated date by session max age seconds.+ * @internal+ */+export function getDefaultCookieExpirationDate(maxAgeSecs: number) {+ return new Date(Date.now() + (maxAgeSecs * 1000));+}++/**+ * Transforms tough-cookie to puppeteer cookie.+ * @param toughCookie Cookie from CookieJar+ * @return Cookie compatible with browser pool+ * @internal+ */+export function toughCookieToBrowserPoolCookie(toughCookie: Cookie): CookieObject {+ return {+ name: toughCookie.key,+ value: toughCookie.value,+ // Puppeteer and Playwright expect 'expires' to be 'Unix time in seconds', not ms+ // If there is no expires date (so defaults to Infinity), we don't provide it to the browsers+ expires: toughCookie.expires === 'Infinity' ? undefined : new Date(toughCookie.expires).getTime() / 1000,+ domain: toughCookie.domain ?? undefined,+ path: toughCookie.path ?? undefined,+ secure: toughCookie.secure,+ httpOnly: toughCookie.httpOnly,+ };+}++/**+ * Transforms browser-pool cookie to tough-cookie.+ * @param cookieObject Cookie object (for instance from the `page.cookies` method).+ * @internal+ */+export function browserPoolCookieToToughCookie(cookieObject: CookieObject, maxAgeSecs: number) {+ const isExpiresValid = cookieObject.expires && typeof cookieObject.expires === 'number' && cookieObject.expires > 0;+ const expires = isExpiresValid ? new Date(cookieObject.expires! * 1000) : getDefaultCookieExpirationDate(maxAgeSecs);+ const domain = typeof cookieObject.domain === 'string' && cookieObject.domain.startsWith('.')+ ? cookieObject.domain.slice(1)+ : cookieObject.domain;++ return new Cookie({+ key: cookieObject.name,+ value: cookieObject.value,+ expires,+ domain,+ path: cookieObject.path,+ secure: cookieObject.secure,+ httpOnly: cookieObject.httpOnly,+ });+}++/**+ * @internal+ * @param cookieString The cookie string to attempt parsing+ * @returns Browser pool compatible cookie, or null if cookie cannot be parsed+ */+export function cookieStringToToughCookie(cookieString: string) {+ const parsed = Cookie.parse(cookieString);++ if (parsed) {+ return toughCookieToBrowserPoolCookie(parsed);+ }++ return null;+}++/**+ * Merges multiple cookie strings. Keys are compared case-sensitively, warning will be logged+ * if we see two cookies with same keys but different casing.+ * @internal+ */+export function mergeCookies(url: string, sourceCookies: string[]): string {+ const jar = new CookieJar();++ // ignore empty cookies+ for (const sourceCookieString of sourceCookies) {+ // ignore empty cookies+ if (!sourceCookieString) continue;++ const cookies = sourceCookieString.split(/ *; */);++ for (const cookieString of cookies) {+ // ignore extra spaces+ if (!cookieString) continue;++ const cookie = Cookie.parse(cookieString)!;+ const similarKeyCookie = jar.getCookiesSync(url).find((c) => {+ return cookie.key !== c.key && cookie.key.toLowerCase() === c.key.toLowerCase();+ });++ if (similarKeyCookie) {+ log.deprecated(`Found cookies with similar name during cookie merging: '${cookie.key}' and '${similarKeyCookie.key}'`);+ }++ jar.setCookieSync(cookie, url);+ }+ }++ return jar.getCookieStringSync(url);+}diff --git a/packages/core/src/crawlers/crawler_utils.ts b/packages/core/src/crawlers/crawler_utils.tsindex c1ae70fe3..779cedcf1 100644--- a/packages/core/src/crawlers/crawler_utils.ts+++ b/packages/core/src/crawlers/crawler_utils.ts@@ -1,6 +1,4 @@import { TimeoutError } from '@apify/timeout';-import { Cookie, CookieJar } from 'tough-cookie';-import { log } from '../log';import type { Session } from '../session_pool/session';/**@@ -13,56 +11,3 @@ export function handleRequestTimeout({ session, errorMessage }: { session?: Sessconst timeoutSecs = Number(timeoutMillis) / 1000;throw new TimeoutError(`Navigation timed out after ${timeoutSecs} seconds.`);}--/**- * Merges multiple cookie strings. Keys are compared case-sensitively, warning will be logged- * if we see two cookies with same keys but different casing.- * @internal- */-export function mergeCookies(url: string, sourceCookies: string[]): string {- const jar = new CookieJar();-- // ignore empty cookies- for (const sourceCookieString of sourceCookies.filter((c) => c)) {- const cookies = sourceCookieString.split(/ *; */).filter((c) => c); // ignore extra spaces-- for (const cookieString of cookies) {- const cookie = Cookie.parse(cookieString)!;- const similarKeyCookie = jar.getCookiesSync(url).find((c) => {- return cookie.key !== c.key && cookie.key.toLowerCase() === c.key.toLowerCase();- });-- if (similarKeyCookie) {- log.deprecated(`Found cookies with similar name during cookie merging: '${cookie.key}' and '${similarKeyCookie.key}'`);- }-- jar.setCookieSync(cookie, url);- }- }-- return jar.getCookieStringSync(url);-}--/**- * @internal- */-export function diffCookies(url: string, cookieString1 = '', cookieString2 = ''): string {- if (cookieString1 === cookieString2 || !cookieString2) {- return '';- }-- if (!cookieString1) {- return cookieString2;- }-- const cookies1 = cookieString1.split(/ *; */).filter((item) => Boolean(item)).map((cookie) => Cookie.parse(cookie)!);- const cookies2 = cookieString2.split(/ *; */).filter((item) => Boolean(item)).map((cookie) => Cookie.parse(cookie)!);-- const added = cookies2.filter((newCookie) => {- return !cookies1.find((oldCookie) => newCookie.toString() === oldCookie.toString());- });- const jar = new CookieJar();- added.forEach((cookie) => jar.setCookieSync(cookie, url));-- return jar.getCookieStringSync(url);-}diff --git a/packages/core/src/index.ts b/packages/core/src/index.tsindex e7a8a7a1f..a493e2fa8 100644--- a/packages/core/src/index.ts+++ b/packages/core/src/index.ts@@ -13,5 +13,6 @@ export * from './serialization';export * from './session_pool';export * from './storages';export * from './validators';+export * from './cookie_utils';export { PseudoUrl } from '@apify/pseudo_url';export { Dictionary, Awaitable, Constructor, StorageClient } from '@crawlee/types';diff --git a/packages/core/src/session_pool/index.ts b/packages/core/src/session_pool/index.tsindex 4f266661a..5d387f855 100644--- a/packages/core/src/session_pool/index.ts+++ b/packages/core/src/session_pool/index.ts@@ -2,4 +2,3 @@ export * from './errors';export * from './events';export * from './session';export * from './session_pool';-export * from './session_utils';diff --git a/packages/core/src/session_pool/session.ts b/packages/core/src/session_pool/session.tsindex 8fdd0a923..7b4c8c273 100644--- a/packages/core/src/session_pool/session.ts+++ b/packages/core/src/session_pool/session.ts@@ -4,11 +4,12 @@ import type { Dictionary, Cookie as CookieObject, BrowserLikeResponse } from '@cimport type { IncomingMessage } from 'node:http';import { EventEmitter } from 'node:events';import ow from 'ow';-import { Cookie, CookieJar } from 'tough-cookie';+import type { Cookie } from 'tough-cookie';+import { CookieJar } from 'tough-cookie';import { STATUS_CODES_BLOCKED } from '../constants';import { log as defaultLog } from '../log';import { EVENT_SESSION_RETIRED } from './events';-import { getCookiesFromResponse } from './session_utils';+import { browserPoolCookieToToughCookie, getCookiesFromResponse, getDefaultCookieExpirationDate, toughCookieToBrowserPoolCookie } from '../cookie_utils';// CONSTANTSconst DEFAULT_SESSION_MAX_AGE_SECS = 3000;@@ -144,7 +145,7 @@ export class Session {log = defaultLog,} = options;- const { expiresAt = this._getDefaultCookieExpirationDate(maxAgeSecs) } = options;+ const { expiresAt = getDefaultCookieExpirationDate(maxAgeSecs) } = options;this.log = log.child({ prefix: 'Session' });@@ -162,8 +163,6 @@ export class Session {this.errorScore = errorScore; // indicates number of markBaded request with the sessionthis.maxUsageCount = maxUsageCount;this.sessionPool = sessionPool;-- this._cookieObjectToTough = this._cookieObjectToTough.bind(this);}/**@@ -308,17 +307,17 @@ export class Session {* ```*/setCookies(cookies: CookieObject[], url: string) {- const normalizedCookies = cookies.map(this._cookieObjectToTough);+ const normalizedCookies = cookies.map((c) => browserPoolCookieToToughCookie(c, this.maxAgeSecs));this._setCookies(normalizedCookies, url);}/**- * Returns cookies in a format compatible with puppeteer and ready to be used with `page.setCookie`.+ * Returns cookies in a format compatible with puppeteer/playwright and ready to be used with `page.setCookie`.* @param url website url. Only cookies stored for this url will be returned*/getCookies(url: string): CookieObject[] {const cookies = this.cookieJar.getCookiesSync(url);- return cookies.map(this._toughCookieToPuppeteer);+ return cookies.map((c) => toughCookieToBrowserPoolCookie(c));}/**@@ -338,46 +337,6 @@ export class Session {this.cookieJar.setCookieSync(rawCookie, url);}- /**- * Transforms puppeteer cookie to tough-cookie.- * @param cookieObject Cookie object (for instance from the `page.cookies` method).- */- protected _cookieObjectToTough(cookieObject: CookieObject): Cookie {- const isExpiresValid = cookieObject.expires && typeof cookieObject.expires === 'number' && cookieObject.expires > 0;- const expires = isExpiresValid ? new Date(cookieObject.expires! * 1000) : this._getDefaultCookieExpirationDate(this.maxAgeSecs);- const domain = typeof cookieObject.domain === 'string' && cookieObject.domain.startsWith('.')- ? cookieObject.domain.slice(1)- : cookieObject.domain;-- return new Cookie({- key: cookieObject.name,- value: cookieObject.value,- expires,- domain,- path: cookieObject.path,- secure: cookieObject.secure,- httpOnly: cookieObject.httpOnly,- });- }-- /**- * Transforms tough-cookie to puppeteer cookie.- * @param toughCookie Cookie from CookieJar- * @return Cookie from Puppeteer- */- protected _toughCookieToPuppeteer(toughCookie: Cookie): CookieObject {- return {- name: toughCookie.key,- value: toughCookie.value,- // Puppeteer and Playwright expect 'expires' to be 'Unix time in seconds', not ms- expires: new Date(toughCookie.expires).getTime() / 1000,- domain: toughCookie.domain,- path: toughCookie.path,- secure: toughCookie.secure,- httpOnly: toughCookie.httpOnly,- } as CookieObject;- }-/*** Sets cookies.*/@@ -399,15 +358,6 @@ export class Session {}}- /**- * Calculate cookie expiration date- * @param maxAgeSecs- * @returns Calculated date by session max age seconds.- */- protected _getDefaultCookieExpirationDate(maxAgeSecs: number): Date {- return new Date(Date.now() + (maxAgeSecs * 1000));- }-/*** Checks if session is not usable. if it is not retires the session.*/diff --git a/packages/core/src/session_pool/session_utils.ts b/packages/core/src/session_pool/session_utils.tsdeleted file mode 100644index bb9b33b1d..000000000--- a/packages/core/src/session_pool/session_utils.ts+++ /dev/null@@ -1,20 +0,0 @@-import type { BrowserLikeResponse, Dictionary } from '@crawlee/types';-import type { IncomingMessage } from 'node:http';-import { Cookie } from 'tough-cookie';-import { CookieParseError } from './errors';--/**- * @internal- */-export function getCookiesFromResponse(response: IncomingMessage | BrowserLikeResponse | { headers: Dictionary<string | string[]> }): Cookie[] {- const headers = typeof response.headers === 'function' ? response.headers() : response.headers;- const cookieHeader = headers['set-cookie'] || '';-- try {- return Array.isArray(cookieHeader)- ? cookieHeader.map((cookie) => Cookie.parse(cookie)!)- : [Cookie.parse(cookieHeader)!];- } catch (e) {- throw new CookieParseError(cookieHeader);- }-}diff --git a/packages/playwright-crawler/package.json b/packages/playwright-crawler/package.jsonindex 1b5049c5f..b3da80af6 100644--- a/packages/playwright-crawler/package.json+++ b/packages/playwright-crawler/package.json@@ -60,8 +60,7 @@"@crawlee/core": "^3.0.1","@crawlee/utils": "^3.0.1","jquery": "^3.6.0",- "ow": "^0.28.1",- "tough-cookie": "^4.0.0"+ "ow": "^0.28.1"},"peerDependencies": {"playwright": "^1.21.1"diff --git a/packages/playwright-crawler/src/internals/playwright-crawler.ts b/packages/playwright-crawler/src/internals/playwright-crawler.tsindex 77fb796ac..1764f123c 100644--- a/packages/playwright-crawler/src/internals/playwright-crawler.ts+++ b/packages/playwright-crawler/src/internals/playwright-crawler.ts@@ -4,7 +4,6 @@ import type { BrowserPoolOptions, PlaywrightController, PlaywrightPlugin } fromimport type { BrowserCrawlerOptions, BrowserCrawlingContext, BrowserRequestHandler, BrowserHook } from '@crawlee/browser';import { BrowserCrawler, Configuration, Router } from '@crawlee/browser';import type { Dictionary } from '@crawlee/types';-import { Cookie } from 'tough-cookie';import type { PlaywrightLaunchContext } from './playwright-launcher';import { PlaywrightLauncher } from './playwright-launcher';import type { DirectNavigationOptions, PlaywrightContextUtils } from './utils/playwright-utils';@@ -216,20 +215,6 @@ export class PlaywrightCrawler extends BrowserCrawler<{ browserPlugins: [Playwriprotected override async _navigationHandler(crawlingContext: PlaywrightCrawlingContext, gotoOptions: DirectNavigationOptions) {return gotoExtended(crawlingContext.page, crawlingContext.request, gotoOptions);}-- protected override async _applyCookies({ session, request, page }: PlaywrightCrawlingContext, preHooksCookies: string, postHooksCookies: string) {- const sessionCookie = session?.getCookies(request.url) ?? [];- const parsedPreHooksCookies = preHooksCookies.split(/ *; */).map((c) => Cookie.parse(c)?.toJSON() as PlaywrightCookie | undefined);- const parsedPostHooksCookies = postHooksCookies.split(/ *; */).map((c) => Cookie.parse(c)?.toJSON() as PlaywrightCookie | undefined);-- await page.context().addCookies(- [- ...sessionCookie,- ...parsedPreHooksCookies,- ...parsedPostHooksCookies,- ].filter((c): c is PlaywrightCookie => c !== undefined),- );- }}/**diff --git a/test/core/crawlers/crawler_utils.test.ts b/test/core/crawlers/crawler_utils.test.tsdeleted file mode 100644index 79fa5cadd..000000000--- a/test/core/crawlers/crawler_utils.test.ts+++ /dev/null@@ -1,28 +0,0 @@-import { diffCookies } from '@crawlee/core';--const cookie1 = 'test_cookie=1';-const cookie2 = 'new_test_cookie=2';--const mergedCookies = [cookie1, cookie2].join(';');--const url = 'https://example.com';--describe('CrawlerUtils', () => {- describe('diffCookies', () => {- test('when no cookies are provided for cookiesString2, return an empty string', () => {- expect(diffCookies(url, cookie1, '')).toEqual('');- });-- test('when no cookies are provided for cookiesString1, return cookiesString2', () => {- expect(diffCookies(url, '', cookie1)).toEqual(cookie1);- });-- test('when new cookies are provided for cookiesString2, return them', () => {- expect(diffCookies(url, cookie1, cookie2)).toEqual(cookie2);- });-- test('when new cookies include previous cookies for cookiesString2, return only the new ones', () => {- expect(diffCookies(url, cookie1, mergedCookies)).toEqual(cookie2);- });- });-});diff --git a/test/e2e/cheerio-initial-cookies/actor/main.js b/test/e2e/cheerio-initial-cookies/actor/main.jsindex 699b87cd3..47aedf7bf 100644--- a/test/e2e/cheerio-initial-cookies/actor/main.js+++ b/test/e2e/cheerio-initial-cookies/actor/main.js@@ -2,18 +2,26 @@ import { Actor } from 'apify';import { CheerioCrawler, Dataset } from '@crawlee/cheerio';import { ApifyStorageLocal } from '@apify/storage-local';-const initialCookies = [+const expectedCookies = [{- name: 'test',- value: 'testing cookies',+ name: 'initial_request',+ value: 'true',},{- name: 'store',- value: 'value store',+ name: 'session',+ value: 'true',},{- name: 'market_place',- value: 'value market place',+ name: 'hook_request',+ value: 'true',+ },+ {+ name: 'got_options_upper_case',+ value: 'true',+ },+ {+ name: 'got_options_lower_case',+ value: 'true',},];@@ -25,11 +33,21 @@ const mainOptions = {await Actor.main(async () => {const crawler = new CheerioCrawler({additionalMimeTypes: ['application/json'],- preNavigationHooks: [({ session, request }) => {- session.setCookies(initialCookies, request.url);+ preNavigationHooks: [({ session, request }, gotOptions) => {+ session.setCookies([+ {+ name: 'session',+ value: 'true',+ },+ ], request.url);+ request.headers.cookie = 'hook_request=true';++ gotOptions.headers ??= {};+ gotOptions.headers.Cookie = 'got_options_upper_case=true';+ gotOptions.headers.cookie = 'got_options_lower_case=true';}],async requestHandler({ json }) {- const initialCookiesLength = initialCookies.length;+ const initialCookiesLength = expectedCookies.length;const cookieString = json.headers.cookie;const pageCookies = cookieString.split(';').map((cookie) => {@@ -38,7 +56,7 @@ await Actor.main(async () => {});let numberOfMatchingCookies = 0;- for (const cookie of initialCookies) {+ for (const cookie of expectedCookies) {if (pageCookies.some((pageCookie) => pageCookie.name === cookie.name && pageCookie.value === cookie.value)) {numberOfMatchingCookies++;}@@ -48,5 +66,12 @@ await Actor.main(async () => {},});- await crawler.run(['https://api.apify.com/v2/browser-info']);+ await crawler.run([+ {+ url: 'https://api.apify.com/v2/browser-info',+ headers: {+ Cookie: 'initial_request=true',+ },+ },+ ]);}, mainOptions);diff --git a/test/e2e/cheerio-initial-cookies/test.mjs b/test/e2e/cheerio-initial-cookies/test.mjsindex b22a8dded..6987142a0 100644--- a/test/e2e/cheerio-initial-cookies/test.mjs+++ b/test/e2e/cheerio-initial-cookies/test.mjs@@ -6,7 +6,7 @@ await initialize(testActorDirname);const { stats, datasetItems } = await runActor(testActorDirname);await expect(stats.requestsFinished === 1, 'All requests finished');-await expect(datasetItems[0].numberOfMatchingCookies === 3, 'Number of page cookies');+await expect(datasetItems[0].numberOfMatchingCookies === 5, 'Number of page cookies');await expect(datasetItems[0].numberOfMatchingCookies === datasetItems[0].initialCookiesLength,`Page cookies match the initial defined cookies. Number of non-matching cookies is `diff --git a/test/e2e/playwright-initial-cookies/actor/.gitignore b/test/e2e/playwright-initial-cookies/actor/.gitignorenew file mode 100644index 000000000..ced7cbfc5--- /dev/null+++ b/test/e2e/playwright-initial-cookies/actor/.gitignore@@ -0,0 +1,7 @@+.idea+.DS_Store+node_modules+package-lock.json+apify_storage+crawlee_storage+storagediff --git a/test/e2e/playwright-initial-cookies/actor/Dockerfile b/test/e2e/playwright-initial-cookies/actor/Dockerfilenew file mode 100644index 000000000..2a1510bd0--- /dev/null+++ b/test/e2e/playwright-initial-cookies/actor/Dockerfile@@ -0,0 +1,22 @@+FROM node:16 AS builder++COPY /packages ./packages+COPY /package*.json ./+RUN npm --quiet set progress=false \+ && npm install --only=prod --no-optional++FROM apify/actor-node-playwright-chrome:16++RUN rm -r node_modules+COPY --from=builder /node_modules ./node_modules+COPY --from=builder /packages ./packages+COPY --from=builder /package*.json ./+COPY /apify.json ./+COPY /main.js ./++RUN echo "Installed NPM packages:" \+ && (npm list --only=prod --no-optional --all || true) \+ && echo "Node.js version:" \+ && node --version \+ && echo "NPM version:" \+ && npm --versiondiff --git a/test/e2e/playwright-initial-cookies/actor/apify.json b/test/e2e/playwright-initial-cookies/actor/apify.jsonnew file mode 100644index 000000000..ca72b5fff--- /dev/null+++ b/test/e2e/playwright-initial-cookies/actor/apify.json@@ -0,0 +1,6 @@+{+ "name": "test-playwright-initial-cookies",+ "version": "0.0",+ "buildTag": "latest",+ "env": null+}diff --git a/test/e2e/playwright-initial-cookies/actor/main.js b/test/e2e/playwright-initial-cookies/actor/main.jsnew file mode 100644index 000000000..04248a91a--- /dev/null+++ b/test/e2e/playwright-initial-cookies/actor/main.js@@ -0,0 +1,62 @@+import { Actor } from 'apify';+import { Dataset, PlaywrightCrawler } from '@crawlee/playwright';+import { ApifyStorageLocal } from '@apify/storage-local';++const expectedCookies = [+ {+ name: 'initial_request',+ value: 'true',+ },+ {+ name: 'session',+ value: 'true',+ },+ {+ name: 'hook_request',+ value: 'true',+ },+];++const mainOptions = {+ exit: Actor.isAtHome(),+ storage: process.env.STORAGE_IMPLEMENTATION === 'LOCAL' ? new ApifyStorageLocal() : undefined,+};++await Actor.main(async () => {+ const crawler = new PlaywrightCrawler({+ preNavigationHooks: [({ session, request }, goToOptions) => {+ session.setCookies([+ {+ name: 'session',+ value: 'true',+ },+ ], request.url);+ request.headers.cookie = 'hook_request=true';++ goToOptions.waitUntil = 'networkidle';+ }],+ async requestHandler({ page }) {+ const initialCookiesLength = expectedCookies.length;++ const pageCookies = await page.context().cookies();++ let numberOfMatchingCookies = 0;+ for (const cookie of expectedCookies) {+ if (pageCookies.some((pageCookie) => pageCookie.name === cookie.name && pageCookie.value === cookie.value)) {+ numberOfMatchingCookies++;+ }+ }++ await Dataset.pushData({ initialCookiesLength, numberOfMatchingCookies });+ },+ });++ await crawler.run([+ {+ url: 'https://api.apify.com/v2/browser-info',+ headers: {+ Cookie: 'initial_request=true',+ },+ },+ ]);+}, mainOptions);diff --git a/test/e2e/playwright-initial-cookies/actor/package.json b/test/e2e/playwright-initial-cookies/actor/package.jsonnew file mode 100644index 000000000..4bdf776e7--- /dev/null+++ b/test/e2e/playwright-initial-cookies/actor/package.json@@ -0,0 +1,21 @@+{+ "name": "test-playwright-initial-cookies",+ "version": "0.0.1",+ "description": "Playwright Test - Initial Cookies",+ "dependencies": {+ "apify": "^3.0.0-beta.75",+ "@apify/storage-local": "^2.1.0",+ "@crawlee/basic": "file:./packages/basic-crawler",+ "@crawlee/browser": "file:./packages/browser-crawler",+ "@crawlee/browser-pool": "file:./packages/browser-pool",+ "@crawlee/core": "file:./packages/core",+ "@crawlee/memory-storage": "file:./packages/memory-storage",+ "@crawlee/playwright": "file:./packages/playwright-crawler",+ "@crawlee/types": "file:./packages/types",+ "@crawlee/utils": "file:./packages/utils",+ "playwright": "*"+ },+ "scripts": { "start": "node main.js" },+ "type": "module",+ "license": "ISC"+}diff --git a/test/e2e/playwright-initial-cookies/test.mjs b/test/e2e/playwright-initial-cookies/test.mjsnew file mode 100644index 000000000..a93bcf543--- /dev/null+++ b/test/e2e/playwright-initial-cookies/test.mjs@@ -0,0 +1,14 @@+import { initialize, getActorTestDir, runActor, expect } from '../tools.mjs';++const testActorDirname = getActorTestDir(import.meta.url);+await initialize(testActorDirname);++const { stats, datasetItems } = await runActor(testActorDirname, 16384);++await expect(stats.requestsFinished === 1, 'All requests finished');+await expect(datasetItems[0].numberOfMatchingCookies === 3, 'Number of page cookies');+await expect(+ datasetItems[0].numberOfMatchingCookies === datasetItems[0].initialCookiesLength,+ `Page cookies match the initial defined cookies. Number of non-matching cookies is `+ + `${datasetItems[0].initialCookiesLength - datasetItems[0].numberOfMatchingCookies}`,+);diff --git a/test/e2e/puppeteer-initial-cookies/actor/main.js b/test/e2e/puppeteer-initial-cookies/actor/main.jsindex fff341937..81689460b 100644--- a/test/e2e/puppeteer-initial-cookies/actor/main.js+++ b/test/e2e/puppeteer-initial-cookies/actor/main.js@@ -2,18 +2,18 @@ import { Actor } from 'apify';import { Dataset, PuppeteerCrawler } from '@crawlee/puppeteer';import { ApifyStorageLocal } from '@apify/storage-local';-const initialCookies = [+const expectedCookies = [{- name: 'test',- value: 'testing cookies',+ name: 'initial_request',+ value: 'true',},{- name: 'store',- value: 'value store',+ name: 'session',+ value: 'true',},{- name: 'market_place',- value: 'value market place',+ name: 'hook_request',+ value: 'true',},];@@ -25,16 +25,23 @@ const mainOptions = {await Actor.main(async () => {const crawler = new PuppeteerCrawler({preNavigationHooks: [({ session, request }, goToOptions) => {- session.setCookies(initialCookies, request.url);+ session.setCookies([+ {+ name: 'session',+ value: 'true',+ },+ ], request.url);+ request.headers.cookie = 'hook_request=true';+goToOptions.waitUntil = ['networkidle2'];}],async requestHandler({ page }) {- const initialCookiesLength = initialCookies.length;+ const initialCookiesLength = expectedCookies.length;const pageCookies = await page.cookies();let numberOfMatchingCookies = 0;- for (const cookie of initialCookies) {+ for (const cookie of expectedCookies) {if (pageCookies.some((pageCookie) => pageCookie.name === cookie.name && pageCookie.value === cookie.value)) {numberOfMatchingCookies++;}@@ -44,5 +51,12 @@ await Actor.main(async () => {},});- await crawler.run(['https://api.apify.com/v2/browser-info']);+ await crawler.run([+ {+ url: 'https://api.apify.com/v2/browser-info',+ headers: {+ Cookie: 'initial_request=true',+ },+ },+ ]);}, mainOptions);diff --git a/test/e2e/run.mjs b/test/e2e/run.mjsindex c2f87204e..d689b63b9 100644--- a/test/e2e/run.mjs+++ b/test/e2e/run.mjs@@ -35,7 +35,7 @@ async function run() {throw new Error(`Unknown storage provided: '${process.env.STORAGE_IMPLEMENTATION}'`);}- execSync(`npm install @apify/storage-local`, { stdio: 'inherit' });+ execSync(`npm install --no-save @apify/storage-local`, { stdio: 'inherit' });console.log(`Running E2E tests with storage implementation '${process.env.STORAGE_IMPLEMENTATION}'`);