Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
fix: declare own `CookieJar` types in `@crawlee/types` (#3978)
Jindřich Bärdd7748d649c4
diff --git a/docs/guides/session_management.mdx b/docs/guides/session_management.mdxindex b5a1ba558..067e8a9c3 100644--- a/docs/guides/session_management.mdx+++ b/docs/guides/session_management.mdx@@ -99,7 +99,7 @@ You can also seed or read cookies yourself. <ApiLink to="core/class/Session#setC```jsconst crawler = new CheerioCrawler({requestHandler: async ({ session, request }) => {- session.setCookie('consent=yes', request.url);+ await session.setCookie('consent=yes', request.url);},});```diff --git a/docs/guides/session_management_basic.ts b/docs/guides/session_management_basic.tsindex 948f75ea0..4a9f46748 100644--- a/docs/guides/session_management_basic.ts+++ b/docs/guides/session_management_basic.ts@@ -17,7 +17,7 @@ const crawler = new BasicCrawler({headers: {// If you want to use the cookieJar.// This way you get the Cookie headers string from session.- Cookie: session?.cookieJar.getCookieStringSync(url) ?? '',+ Cookie: (await session?.cookieJar.getCookieString(url)) ?? '',},});let response;diff --git a/docs/public-api/crawlee-core.api.md b/docs/public-api/crawlee-core.api.mdindex 35768796e..0d72df75b 100644--- a/docs/public-api/crawlee-core.api.md+++ b/docs/public-api/crawlee-core.api.md@@ -1728,7 +1728,7 @@ export class Session implements ISession {// (undocumented)get fingerprint(): SessionFingerprint | undefined;set fingerprint(fingerprint: SessionFingerprint | undefined);- getCookieString(url: string): string;+ getCookieString(url: string): Promise<string>;getState(): SessionState;// (undocumented)readonly id: string;@@ -1746,7 +1746,7 @@ export class Session implements ISession {get proxyInfo(): ProxyInfo | undefined;retire(): void;get retired(): boolean;- setCookie(rawCookie: string, url: string): void;+ setCookie(rawCookie: string, url: string): Promise<void>;// (undocumented)get usageCount(): number;// (undocumented)diff --git a/docs/public-api/crawlee-types.api.md b/docs/public-api/crawlee-types.api.mdindex d7918d2e7..87cac4e4b 100644--- a/docs/public-api/crawlee-types.api.md+++ b/docs/public-api/crawlee-types.api.md@@ -4,9 +4,7 @@```ts-import type { CookieJar } from 'tough-cookie';import type { Readable } from 'node:stream';-import type { SerializedCookieJar } from 'tough-cookie';// @public (undocumented)export type AllowedHttpMethods = 'GET' | 'HEAD' | 'POST' | 'PUT' | 'DELETE' | 'TRACE' | 'OPTIONS' | 'CONNECT' | 'PATCH' | 'get' | 'head' | 'post' | 'put' | 'delete' | 'trace' | 'options' | 'connect' | 'patch';@@ -49,6 +47,52 @@ export interface Cookie {value: string;}+// @public+export interface CookieJar {+ // (undocumented)+ clone(): Promise<CookieJar>;+ // (undocumented)+ getCookies(url: string | URL, options?: CookieJarGetCookiesOptions): Promise<SessionCookie[]>;+ // (undocumented)+ getCookieString(url: string | URL, options?: CookieJarGetCookiesOptions): Promise<string>;+ // (undocumented)+ getSetCookieStrings(url: string | URL, options?: CookieJarGetCookiesOptions): Promise<string[] | undefined>;+ // (undocumented)+ serialize(): Promise<SerializedCookieJar>;+ // (undocumented)+ setCookie(cookie: string | SessionCookie, url: string | URL, options?: CookieJarSetCookieOptions): Promise<SessionCookie | undefined>;+ // (undocumented)+ toJSON(): SerializedCookieJar | undefined;+}++// @public+export interface CookieJarGetCookiesOptions {+ // (undocumented)+ allPaths?: boolean;+ // (undocumented)+ expire?: boolean;+ // (undocumented)+ http?: boolean;+ // (undocumented)+ sameSiteContext?: 'none' | 'lax' | 'strict';+ // (undocumented)+ sort?: boolean;+}++// @public+export interface CookieJarSetCookieOptions {+ // (undocumented)+ http?: boolean;+ // (undocumented)+ ignoreError?: boolean;+ // (undocumented)+ loose?: boolean;+ // (undocumented)+ now?: Date;+ // (undocumented)+ sameSiteContext?: 'strict' | 'lax' | 'none';+}+// @publicexport interface CrawleeLogger {child(options: Partial<CrawleeLoggerOptions>): CrawleeLogger;@@ -403,6 +447,80 @@ export interface SendRequestOptions {timeoutMillis?: number;}+// @public+export interface SerializedCookieJar {+ // (undocumented)+ [key: string]: unknown;+ // (undocumented)+ cookies: Record<string, unknown>[];+ // (undocumented)+ rejectPublicSuffixes: boolean;+ // (undocumented)+ storeType: string | null;+ // (undocumented)+ version: string;+}++// @public+export interface SessionCookie {+ // (undocumented)+ canonicalizedDomain(): string | undefined;+ // (undocumented)+ cdomain(): string | undefined;+ // (undocumented)+ clone(): SessionCookie | undefined;+ // (undocumented)+ cookieString(): string;+ // (undocumented)+ creation: Date | 'Infinity' | null;+ // (undocumented)+ creationIndex: number;+ // (undocumented)+ domain: string | null;+ // (undocumented)+ expires: Date | 'Infinity' | null;+ // (undocumented)+ expiryDate(now?: Date): Date | undefined;+ // (undocumented)+ expiryTime(now?: Date): number | undefined;+ // (undocumented)+ extensions: string[] | null;+ // (undocumented)+ hostOnly: boolean | null;+ // (undocumented)+ httpOnly: boolean;+ // (undocumented)+ isPersistent(): boolean;+ // (undocumented)+ key: string;+ // (undocumented)+ lastAccessed: Date | 'Infinity' | null;+ // (undocumented)+ maxAge: number | 'Infinity' | '-Infinity' | null;+ // (undocumented)+ path: string | null;+ // (undocumented)+ pathIsDefault: boolean | null;+ // (undocumented)+ sameSite: string | undefined;+ // (undocumented)+ secure: boolean;+ // (undocumented)+ setExpires(exp: string | Date): void;+ // (undocumented)+ setMaxAge(age: number): void;+ // (undocumented)+ toJSON(): Record<string, unknown>;+ // (undocumented)+ toString(): string;+ // (undocumented)+ TTL(now?: number): number;+ // (undocumented)+ validate(): boolean;+ // (undocumented)+ value: string;+}+// @publicexport interface SessionFingerprint {browser?: 'chrome' | 'firefox' | 'safari' | 'edge';diff --git a/packages/browser-crawler/src/internals/browser-crawler.ts b/packages/browser-crawler/src/internals/browser-crawler.tsindex 5cc17696e..fd959fc69 100644--- a/packages/browser-crawler/src/internals/browser-crawler.ts+++ b/packages/browser-crawler/src/internals/browser-crawler.ts@@ -728,7 +728,7 @@ export abstract class BrowserCrawler<(await crawlingContext.page.url()) || crawlingContext.request.loadedUrl || crawlingContext.request.url;for (const cookie of cookies) {try {- crawlingContext.session.cookieJar.setCookieSync(browserPoolCookieToToughCookie(cookie), url, {+ await crawlingContext.session.cookieJar.setCookie(browserPoolCookieToToughCookie(cookie), url, {ignoreError: false,});} catch (e) {@@ -774,7 +774,9 @@ export abstract class BrowserCrawler<preHooksCookies: string,postHooksCookies: string,) {- const sessionCookie = session?.cookieJar.getCookiesSync(request.url).map(toughCookieToBrowserPoolCookie) ?? [];+ const sessionCookie = session+ ? (await session.cookieJar.getCookies(request.url)).map(toughCookieToBrowserPoolCookie)+ : [];const parsedPreHooksCookies = preHooksCookies.split(/ *; */).map((c) => cookieStringToToughCookie(c));const parsedPostHooksCookies = postHooksCookies.split(/ *; */).map((c) => cookieStringToToughCookie(c));diff --git a/packages/core/src/session_pool/session.ts b/packages/core/src/session_pool/session.tsindex 27c627c72..b26826408 100644--- a/packages/core/src/session_pool/session.ts+++ b/packages/core/src/session_pool/session.ts@@ -314,16 +314,16 @@ export class Session implements ISession {* a cookie header or elsewhere.* @returns Represents `Cookie` header.*/- getCookieString(url: string): string {- return this.cookieJar.getCookieStringSync(url, {});+ async getCookieString(url: string): Promise<string> {+ return this.cookieJar.getCookieString(url, {});}/*** Sets a cookie within this session for the specific URL.*/- setCookie(rawCookie: string, url: string): void {+ async setCookie(rawCookie: string, url: string): Promise<void> {try {- this.cookieJar.setCookieSync(rawCookie, url);+ await this.cookieJar.setCookie(rawCookie, url);} catch (e) {this.#log.warning('Could not set cookie.', { url, error: (e as Error).message });}diff --git a/packages/http-client/src/base-http-client.ts b/packages/http-client/src/base-http-client.tsindex daec82dba..15c557859 100644--- a/packages/http-client/src/base-http-client.ts+++ b/packages/http-client/src/base-http-client.ts@@ -76,7 +76,7 @@ export abstract class BaseHttpClient implements BaseHttpClientInterface {.filter(Boolean).map((pair) => merged.setCookie(pair, request.url)),);- const cookieString = merged.getCookieStringSync(request.url);+ const cookieString = await merged.getCookieString(request.url);if (cookieString) {request.headers.set('cookie', cookieString);diff --git a/packages/http-crawler/src/internals/http-crawler.ts b/packages/http-crawler/src/internals/http-crawler.tsindex e0ad1f3f7..d330b061e 100644--- a/packages/http-crawler/src/internals/http-crawler.ts+++ b/packages/http-crawler/src/internals/http-crawler.ts@@ -614,7 +614,9 @@ export class HttpCrawler<for (const cookie of getCookiesFromResponse(response)) {if (!cookie) continue;try {- crawlingContext.session.cookieJar.setCookieSync(cookie, response.url, { ignoreError: false });+ await crawlingContext.session.cookieJar.setCookie(cookie, response.url, {+ ignoreError: false,+ });} catch (e) {this.log.debug(`Could not set cookie: ${(e as Error).message}`);}diff --git a/packages/types/package.json b/packages/types/package.jsonindex 82066812e..b0998d5f6 100644--- a/packages/types/package.json+++ b/packages/types/package.json@@ -43,7 +43,6 @@"access": "public"},"dependencies": {- "tough-cookie": "^6.0.0","tslib": "^2.8.1"}}diff --git a/packages/types/src/cookies.ts b/packages/types/src/cookies.tsnew file mode 100644index 000000000..ab258e1f4--- /dev/null+++ b/packages/types/src/cookies.ts@@ -0,0 +1,90 @@+/**+ * Cookie shape used by {@apilink CookieJar}. Structurally compatible with `tough-cookie`'s+ * `Cookie` class, so a `tough-cookie` cookie satisfies it without `@crawlee/types` depending+ * on `tough-cookie` itself.+ */+export interface SessionCookie {+ key: string;+ value: string;+ expires: Date | 'Infinity' | null;+ maxAge: number | 'Infinity' | '-Infinity' | null;+ domain: string | null;+ path: string | null;+ secure: boolean;+ httpOnly: boolean;+ extensions: string[] | null;+ creation: Date | 'Infinity' | null;+ creationIndex: number;+ hostOnly: boolean | null;+ pathIsDefault: boolean | null;+ lastAccessed: Date | 'Infinity' | null;+ sameSite: string | undefined;+ toJSON(): Record<string, unknown>;+ clone(): SessionCookie | undefined;+ validate(): boolean;+ setExpires(exp: string | Date): void;+ setMaxAge(age: number): void;+ cookieString(): string;+ toString(): string;+ TTL(now?: number): number;+ expiryTime(now?: Date): number | undefined;+ expiryDate(now?: Date): Date | undefined;+ isPersistent(): boolean;+ canonicalizedDomain(): string | undefined;+ cdomain(): string | undefined;+}++/**+ * Options for {@apilink CookieJar.setCookie}. Structurally compatible with `tough-cookie`'s+ * `SetCookieOptions`.+ */+export interface CookieJarSetCookieOptions {+ loose?: boolean;+ sameSiteContext?: 'strict' | 'lax' | 'none';+ ignoreError?: boolean;+ http?: boolean;+ now?: Date;+}++/**+ * Options for {@apilink CookieJar}'s cookie-reading methods. Structurally compatible with+ * `tough-cookie`'s `GetCookiesOptions`.+ */+export interface CookieJarGetCookiesOptions {+ http?: boolean;+ expire?: boolean;+ allPaths?: boolean;+ sameSiteContext?: 'none' | 'lax' | 'strict';+ sort?: boolean;+}++/**+ * Cookie jar contract, structurally compatible with `tough-cookie`'s `CookieJar` — which is+ * what backs {@apilink Session} by default — so `@crawlee/types` does not have to depend on+ * `tough-cookie` for it.+ */+export interface CookieJar {+ setCookie(+ cookie: string | SessionCookie,+ url: string | URL,+ options?: CookieJarSetCookieOptions,+ ): Promise<SessionCookie | undefined>;+ getCookies(url: string | URL, options?: CookieJarGetCookiesOptions): Promise<SessionCookie[]>;+ getCookieString(url: string | URL, options?: CookieJarGetCookiesOptions): Promise<string>;+ getSetCookieStrings(url: string | URL, options?: CookieJarGetCookiesOptions): Promise<string[] | undefined>;+ serialize(): Promise<SerializedCookieJar>;+ toJSON(): SerializedCookieJar | undefined;+ clone(): Promise<CookieJar>;+}++/**+ * JSON representation of a {@apilink CookieJar}. Structurally compatible with `tough-cookie`'s+ * `SerializedCookieJar`.+ */+export interface SerializedCookieJar {+ version: string;+ storeType: string | null;+ rejectPublicSuffixes: boolean;+ cookies: Record<string, unknown>[];+ [key: string]: unknown;+}diff --git a/packages/types/src/http-client.ts b/packages/types/src/http-client.tsindex 5362750f0..b7e6e0d2f 100644--- a/packages/types/src/http-client.ts+++ b/packages/types/src/http-client.ts@@ -1,7 +1,6 @@import type { Readable } from 'node:stream';-import type { CookieJar } from 'tough-cookie';-+import type { CookieJar } from './cookies.js';import type { ISession } from './session.js';import type { AllowedHttpMethods } from './utility-types.js';diff --git a/packages/types/src/index.ts b/packages/types/src/index.tsindex e4af087a5..c0da6dc91 100644--- a/packages/types/src/index.ts+++ b/packages/types/src/index.ts@@ -2,6 +2,7 @@ export type * from './status-message.js';export type * from './storages.js';export type * from './utility-types.js';export type * from './browser.js';+export type * from './cookies.js';export type * from './http-client.js';export type * from './session.js';export type * from './logger.js';diff --git a/packages/types/src/session.ts b/packages/types/src/session.tsindex bae27cf24..70618eff6 100644--- a/packages/types/src/session.ts+++ b/packages/types/src/session.ts@@ -1,4 +1,4 @@-import type { CookieJar, SerializedCookieJar } from 'tough-cookie';+import type { CookieJar, SerializedCookieJar } from './cookies.js';/*** The main purpose of the ProxyInfo object is to provide informationdiff --git a/pnpm-lock.yaml b/pnpm-lock.yamlindex c5c5b6d95..4049d26ca 100644--- a/pnpm-lock.yaml+++ b/pnpm-lock.yaml@@ -895,9 +895,6 @@ importers:packages/types:dependencies:- tough-cookie:- specifier: ^6.0.0- version: 6.0.1tslib:specifier: ^2.8.1version: 2.8.1diff --git a/test/core/crawlers/browser_crawler.test.ts b/test/core/crawlers/browser_crawler.test.tsindex 9aa3460ca..462ba9bbb 100644--- a/test/core/crawlers/browser_crawler.test.ts+++ b/test/core/crawlers/browser_crawler.test.ts@@ -644,8 +644,7 @@ describe('BrowserCrawler', () => {requestList,saveResponseCookies: true,requestHandler: async ({ session, request }) => {- loadedCookies.push(session.cookieJar.getCookieStringSync(request.url));- return Promise.resolve();+ loadedCookies.push(await session.cookieJar.getCookieString(request.url));},preNavigationHooks: [async ({ session, page }) => {@@ -696,7 +695,7 @@ describe('BrowserCrawler', () => {maxPoolSize: 1,}),requestHandler: async ({ page, session, request }) => {- cookieStrings.push(session.cookieJar.getCookieStringSync(request.url));+ cookieStrings.push(await session.cookieJar.getCookieString(request.url));if (request.url.includes('cookie-1')) {const hostname = new URL(request.loadedUrl || request.url).hostname;diff --git a/test/core/crawlers/cheerio_crawler.test.ts b/test/core/crawlers/cheerio_crawler.test.tsindex aaa733282..d019c7b29 100644--- a/test/core/crawlers/cheerio_crawler.test.ts+++ b/test/core/crawlers/cheerio_crawler.test.ts@@ -1253,10 +1253,10 @@ describe('CheerioCrawler', () => {maxPoolSize: 1,}),preNavigationHooks: [- ({ session, request }) => {+ async ({ session, request }) => {// this should get overriden by the server- session.cookieJar.setCookieSync('foo=bar1', request.url);- session.cookieJar.setCookieSync('other=cookie1', request.url);+ await session.cookieJar.setCookie('foo=bar1', request.url);+ await session.cookieJar.setCookie('other=cookie1', request.url);request.headers ??= {};request.headers.cookie += '; coo=kie';@@ -1505,7 +1505,7 @@ describe('CheerioCrawler', () => {},]),requestHandler: async ({ session, request }) => {- sessionCookies.push(session.cookieJar.getCookieStringSync(request.url));+ sessionCookies.push(await session.cookieJar.getCookieString(request.url));},});@@ -1522,8 +1522,8 @@ describe('CheerioCrawler', () => {maxConcurrency: 1,requestList: await RequestList.open(null, [`${serverAddress}/special/get-cookies`]),preNavigationHooks: [- ({ session, request }) => {- session.cookieJar.setCookieSync('manual=fromHook', request.url);+ async ({ session, request }) => {+ await session.cookieJar.setCookie('manual=fromHook', request.url);},],requestHandler: ({ json }) => {diff --git a/test/core/crawlers/puppeteer_crawler.test.ts b/test/core/crawlers/puppeteer_crawler.test.tsindex 054c5f84c..a5344be85 100644--- a/test/core/crawlers/puppeteer_crawler.test.ts+++ b/test/core/crawlers/puppeteer_crawler.test.ts@@ -319,15 +319,15 @@ describe('PuppeteerCrawler', () => {saveResponseCookies: true,sessionPool: new SessionPool({- createSessionFunction: () => {+ createSessionFunction: async () => {const session = new Session();- session.cookieJar.setCookieSync('example_cookie_name=example_cookie_value', serverUrl);+ await session.cookieJar.setCookie('example_cookie_name=example_cookie_value', serverUrl);return session;},}),requestHandler: async ({ page, session }) => {pageCookies = await page.cookies().then((cks) => cks.map((c) => `${c.name}=${c.value}`).join('; '));- sessionCookies = session!.cookieJar.getCookieStringSync(serverUrl);+ sessionCookies = await session!.cookieJar.getCookieString(serverUrl);},});diff --git a/test/core/session_pool/session.test.ts b/test/core/session_pool/session.test.tsindex 98ddac123..1fc83a41b 100644--- a/test/core/session_pool/session.test.ts+++ b/test/core/session_pool/session.test.ts@@ -143,9 +143,9 @@ describe('Session - testing session behaviour', () => {expect(session.cookieJar.setCookie).toBeDefined();});- test('setCookie does not throw on malformed raw cookie string', () => {+ test('setCookie does not throw on malformed raw cookie string', async () => {session = new Session();- expect(() => session.setCookie('garbled!!!@#$%nonsense', 'https://www.example.com')).not.toThrow();+ await expect(session.setCookie('garbled!!!@#$%nonsense', 'https://www.example.com')).resolves.not.toThrow();});test('retired state survives a getState() / new Session() round-trip', () => {@@ -168,11 +168,14 @@ describe('Session - testing session behaviour', () => {expect(reinitialized.isUsable()).toBe(false);});- test('should correctly persist and init cookieJar', () => {+ test('should correctly persist and init cookieJar', async () => {const newSession = new Session();const url = 'https://example.com';- newSession.cookieJar.setCookieSync('CSRF=e8b667; Domain=example.com; Secure', url);- newSession.cookieJar.setCookieSync('id=a3fWa; Expires=Wed, 21 Oct 2099 07:28:00 GMT; Domain=example.com', url);+ await newSession.cookieJar.setCookie('CSRF=e8b667; Domain=example.com; Secure', url);+ await newSession.cookieJar.setCookie(+ 'id=a3fWa; Expires=Wed, 21 Oct 2099 07:28:00 GMT; Domain=example.com',+ url,+ );const old = newSession.getState();@@ -183,6 +186,6 @@ describe('Session - testing session behaviour', () => {// @ts-expect-error string -> Date for createdAt has been overriddenconst reinitializedSession = new Session({ ...old });- expect(reinitializedSession.getCookieString(url)).toEqual('CSRF=e8b667; id=a3fWa');+ await expect(reinitializedSession.getCookieString(url)).resolves.toEqual('CSRF=e8b667; id=a3fWa');});});diff --git a/test/core/session_pool/session_pool.test.ts b/test/core/session_pool/session_pool.test.tsindex 0319255f4..7c1e489b7 100644--- a/test/core/session_pool/session_pool.test.ts+++ b/test/core/session_pool/session_pool.test.ts@@ -114,8 +114,8 @@ describe('SessionPool - testing session pool', () => {test('get state should work', async () => {const url = 'https://example.com';const newSession = await sessionPool.getSession();- newSession?.cookieJar.setCookieSync('cookie1=my-cookie', url);- newSession?.cookieJar.setCookieSync('cookie2=your-cookie', url);+ await newSession?.cookieJar.setCookie('cookie1=my-cookie', url);+ await newSession?.cookieJar.setCookie('cookie2=your-cookie', url);const state = await sessionPool.getState();expect(state).toBeInstanceOf(Object);