Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

fix: declare own `CookieJar` types in `@crawlee/types` (#3978)

Jindřich Bärdd7748d649c4
diff --git a/docs/guides/session_management.mdx b/docs/guides/session_management.mdx
index b5a1ba558..067e8a9c3 100644
--- a/docs/guides/session_management.mdx
+++ b/docs/guides/session_management.mdx
@@ -99,7 +99,7 @@ You can also seed or read cookies yourself. <ApiLink to="core/class/Session#setC
```js
const crawler = new CheerioCrawler({
requestHandler: async ({ session, request }) => {
- session.setCookie('consent=yes', request.url);
+ await session.setCookie('consent=yes', request.url);
},
});
```
diff --git a/docs/guides/session_management_basic.ts b/docs/guides/session_management_basic.ts
index 948f75ea0..4a9f46748 100644
--- a/docs/guides/session_management_basic.ts
+++ b/docs/guides/session_management_basic.ts
@@ -17,7 +17,7 @@ const crawler = new BasicCrawler({
headers: {
// If you want to use the cookieJar.
// This way you get the Cookie headers string from session.
- Cookie: session?.cookieJar.getCookieStringSync(url) ?? '',
+ Cookie: (await session?.cookieJar.getCookieString(url)) ?? '',
},
});
let response;
diff --git a/docs/public-api/crawlee-core.api.md b/docs/public-api/crawlee-core.api.md
index 35768796e..0d72df75b 100644
--- a/docs/public-api/crawlee-core.api.md
+++ b/docs/public-api/crawlee-core.api.md
@@ -1728,7 +1728,7 @@ export class Session implements ISession {
// (undocumented)
get fingerprint(): SessionFingerprint | undefined;
set fingerprint(fingerprint: SessionFingerprint | undefined);
- getCookieString(url: string): string;
+ getCookieString(url: string): Promise<string>;
getState(): SessionState;
// (undocumented)
readonly id: string;
@@ -1746,7 +1746,7 @@ export class Session implements ISession {
get proxyInfo(): ProxyInfo | undefined;
retire(): void;
get retired(): boolean;
- setCookie(rawCookie: string, url: string): void;
+ setCookie(rawCookie: string, url: string): Promise<void>;
// (undocumented)
get usageCount(): number;
// (undocumented)
diff --git a/docs/public-api/crawlee-types.api.md b/docs/public-api/crawlee-types.api.md
index d7918d2e7..87cac4e4b 100644
--- a/docs/public-api/crawlee-types.api.md
+++ b/docs/public-api/crawlee-types.api.md
@@ -4,9 +4,7 @@
```ts
-import type { CookieJar } from 'tough-cookie';
import type { Readable } from 'node:stream';
-import type { SerializedCookieJar } from 'tough-cookie';
// @public (undocumented)
export type AllowedHttpMethods = 'GET' | 'HEAD' | 'POST' | 'PUT' | 'DELETE' | 'TRACE' | 'OPTIONS' | 'CONNECT' | 'PATCH' | 'get' | 'head' | 'post' | 'put' | 'delete' | 'trace' | 'options' | 'connect' | 'patch';
@@ -49,6 +47,52 @@ export interface Cookie {
value: string;
}
+// @public
+export interface CookieJar {
+ // (undocumented)
+ clone(): Promise<CookieJar>;
+ // (undocumented)
+ getCookies(url: string | URL, options?: CookieJarGetCookiesOptions): Promise<SessionCookie[]>;
+ // (undocumented)
+ getCookieString(url: string | URL, options?: CookieJarGetCookiesOptions): Promise<string>;
+ // (undocumented)
+ getSetCookieStrings(url: string | URL, options?: CookieJarGetCookiesOptions): Promise<string[] | undefined>;
+ // (undocumented)
+ serialize(): Promise<SerializedCookieJar>;
+ // (undocumented)
+ setCookie(cookie: string | SessionCookie, url: string | URL, options?: CookieJarSetCookieOptions): Promise<SessionCookie | undefined>;
+ // (undocumented)
+ toJSON(): SerializedCookieJar | undefined;
+}
+
+// @public
+export interface CookieJarGetCookiesOptions {
+ // (undocumented)
+ allPaths?: boolean;
+ // (undocumented)
+ expire?: boolean;
+ // (undocumented)
+ http?: boolean;
+ // (undocumented)
+ sameSiteContext?: 'none' | 'lax' | 'strict';
+ // (undocumented)
+ sort?: boolean;
+}
+
+// @public
+export interface CookieJarSetCookieOptions {
+ // (undocumented)
+ http?: boolean;
+ // (undocumented)
+ ignoreError?: boolean;
+ // (undocumented)
+ loose?: boolean;
+ // (undocumented)
+ now?: Date;
+ // (undocumented)
+ sameSiteContext?: 'strict' | 'lax' | 'none';
+}
+
// @public
export interface CrawleeLogger {
child(options: Partial<CrawleeLoggerOptions>): CrawleeLogger;
@@ -403,6 +447,80 @@ export interface SendRequestOptions {
timeoutMillis?: number;
}
+// @public
+export interface SerializedCookieJar {
+ // (undocumented)
+ [key: string]: unknown;
+ // (undocumented)
+ cookies: Record<string, unknown>[];
+ // (undocumented)
+ rejectPublicSuffixes: boolean;
+ // (undocumented)
+ storeType: string | null;
+ // (undocumented)
+ version: string;
+}
+
+// @public
+export interface SessionCookie {
+ // (undocumented)
+ canonicalizedDomain(): string | undefined;
+ // (undocumented)
+ cdomain(): string | undefined;
+ // (undocumented)
+ clone(): SessionCookie | undefined;
+ // (undocumented)
+ cookieString(): string;
+ // (undocumented)
+ creation: Date | 'Infinity' | null;
+ // (undocumented)
+ creationIndex: number;
+ // (undocumented)
+ domain: string | null;
+ // (undocumented)
+ expires: Date | 'Infinity' | null;
+ // (undocumented)
+ expiryDate(now?: Date): Date | undefined;
+ // (undocumented)
+ expiryTime(now?: Date): number | undefined;
+ // (undocumented)
+ extensions: string[] | null;
+ // (undocumented)
+ hostOnly: boolean | null;
+ // (undocumented)
+ httpOnly: boolean;
+ // (undocumented)
+ isPersistent(): boolean;
+ // (undocumented)
+ key: string;
+ // (undocumented)
+ lastAccessed: Date | 'Infinity' | null;
+ // (undocumented)
+ maxAge: number | 'Infinity' | '-Infinity' | null;
+ // (undocumented)
+ path: string | null;
+ // (undocumented)
+ pathIsDefault: boolean | null;
+ // (undocumented)
+ sameSite: string | undefined;
+ // (undocumented)
+ secure: boolean;
+ // (undocumented)
+ setExpires(exp: string | Date): void;
+ // (undocumented)
+ setMaxAge(age: number): void;
+ // (undocumented)
+ toJSON(): Record<string, unknown>;
+ // (undocumented)
+ toString(): string;
+ // (undocumented)
+ TTL(now?: number): number;
+ // (undocumented)
+ validate(): boolean;
+ // (undocumented)
+ value: string;
+}
+
// @public
export interface SessionFingerprint {
browser?: 'chrome' | 'firefox' | 'safari' | 'edge';
diff --git a/packages/browser-crawler/src/internals/browser-crawler.ts b/packages/browser-crawler/src/internals/browser-crawler.ts
index 5cc17696e..fd959fc69 100644
--- a/packages/browser-crawler/src/internals/browser-crawler.ts
+++ b/packages/browser-crawler/src/internals/browser-crawler.ts
@@ -728,7 +728,7 @@ export abstract class BrowserCrawler<
(await crawlingContext.page.url()) || crawlingContext.request.loadedUrl || crawlingContext.request.url;
for (const cookie of cookies) {
try {
- crawlingContext.session.cookieJar.setCookieSync(browserPoolCookieToToughCookie(cookie), url, {
+ await crawlingContext.session.cookieJar.setCookie(browserPoolCookieToToughCookie(cookie), url, {
ignoreError: false,
});
} catch (e) {
@@ -774,7 +774,9 @@ export abstract class BrowserCrawler<
preHooksCookies: string,
postHooksCookies: string,
) {
- const sessionCookie = session?.cookieJar.getCookiesSync(request.url).map(toughCookieToBrowserPoolCookie) ?? [];
+ const sessionCookie = session
+ ? (await session.cookieJar.getCookies(request.url)).map(toughCookieToBrowserPoolCookie)
+ : [];
const parsedPreHooksCookies = preHooksCookies.split(/ *; */).map((c) => cookieStringToToughCookie(c));
const parsedPostHooksCookies = postHooksCookies.split(/ *; */).map((c) => cookieStringToToughCookie(c));
diff --git a/packages/core/src/session_pool/session.ts b/packages/core/src/session_pool/session.ts
index 27c627c72..b26826408 100644
--- a/packages/core/src/session_pool/session.ts
+++ b/packages/core/src/session_pool/session.ts
@@ -314,16 +314,16 @@ export class Session implements ISession {
* a cookie header or elsewhere.
* @returns Represents `Cookie` header.
*/
- getCookieString(url: string): string {
- return this.cookieJar.getCookieStringSync(url, {});
+ async getCookieString(url: string): Promise<string> {
+ return this.cookieJar.getCookieString(url, {});
}
/**
* Sets a cookie within this session for the specific URL.
*/
- setCookie(rawCookie: string, url: string): void {
+ async setCookie(rawCookie: string, url: string): Promise<void> {
try {
- this.cookieJar.setCookieSync(rawCookie, url);
+ await this.cookieJar.setCookie(rawCookie, url);
} catch (e) {
this.#log.warning('Could not set cookie.', { url, error: (e as Error).message });
}
diff --git a/packages/http-client/src/base-http-client.ts b/packages/http-client/src/base-http-client.ts
index daec82dba..15c557859 100644
--- a/packages/http-client/src/base-http-client.ts
+++ b/packages/http-client/src/base-http-client.ts
@@ -76,7 +76,7 @@ export abstract class BaseHttpClient implements BaseHttpClientInterface {
.filter(Boolean)
.map((pair) => merged.setCookie(pair, request.url)),
);
- const cookieString = merged.getCookieStringSync(request.url);
+ const cookieString = await merged.getCookieString(request.url);
if (cookieString) {
request.headers.set('cookie', cookieString);
diff --git a/packages/http-crawler/src/internals/http-crawler.ts b/packages/http-crawler/src/internals/http-crawler.ts
index e0ad1f3f7..d330b061e 100644
--- a/packages/http-crawler/src/internals/http-crawler.ts
+++ b/packages/http-crawler/src/internals/http-crawler.ts
@@ -614,7 +614,9 @@ export class HttpCrawler<
for (const cookie of getCookiesFromResponse(response)) {
if (!cookie) continue;
try {
- crawlingContext.session.cookieJar.setCookieSync(cookie, response.url, { ignoreError: false });
+ await crawlingContext.session.cookieJar.setCookie(cookie, response.url, {
+ ignoreError: false,
+ });
} catch (e) {
this.log.debug(`Could not set cookie: ${(e as Error).message}`);
}
diff --git a/packages/types/package.json b/packages/types/package.json
index 82066812e..b0998d5f6 100644
--- a/packages/types/package.json
+++ b/packages/types/package.json
@@ -43,7 +43,6 @@
"access": "public"
},
"dependencies": {
- "tough-cookie": "^6.0.0",
"tslib": "^2.8.1"
}
}
diff --git a/packages/types/src/cookies.ts b/packages/types/src/cookies.ts
new file mode 100644
index 000000000..ab258e1f4
--- /dev/null
+++ b/packages/types/src/cookies.ts
@@ -0,0 +1,90 @@
+/**
+ * Cookie shape used by {@apilink CookieJar}. Structurally compatible with `tough-cookie`'s
+ * `Cookie` class, so a `tough-cookie` cookie satisfies it without `@crawlee/types` depending
+ * on `tough-cookie` itself.
+ */
+export interface SessionCookie {
+ key: string;
+ value: string;
+ expires: Date | 'Infinity' | null;
+ maxAge: number | 'Infinity' | '-Infinity' | null;
+ domain: string | null;
+ path: string | null;
+ secure: boolean;
+ httpOnly: boolean;
+ extensions: string[] | null;
+ creation: Date | 'Infinity' | null;
+ creationIndex: number;
+ hostOnly: boolean | null;
+ pathIsDefault: boolean | null;
+ lastAccessed: Date | 'Infinity' | null;
+ sameSite: string | undefined;
+ toJSON(): Record<string, unknown>;
+ clone(): SessionCookie | undefined;
+ validate(): boolean;
+ setExpires(exp: string | Date): void;
+ setMaxAge(age: number): void;
+ cookieString(): string;
+ toString(): string;
+ TTL(now?: number): number;
+ expiryTime(now?: Date): number | undefined;
+ expiryDate(now?: Date): Date | undefined;
+ isPersistent(): boolean;
+ canonicalizedDomain(): string | undefined;
+ cdomain(): string | undefined;
+}
+
+/**
+ * Options for {@apilink CookieJar.setCookie}. Structurally compatible with `tough-cookie`'s
+ * `SetCookieOptions`.
+ */
+export interface CookieJarSetCookieOptions {
+ loose?: boolean;
+ sameSiteContext?: 'strict' | 'lax' | 'none';
+ ignoreError?: boolean;
+ http?: boolean;
+ now?: Date;
+}
+
+/**
+ * Options for {@apilink CookieJar}'s cookie-reading methods. Structurally compatible with
+ * `tough-cookie`'s `GetCookiesOptions`.
+ */
+export interface CookieJarGetCookiesOptions {
+ http?: boolean;
+ expire?: boolean;
+ allPaths?: boolean;
+ sameSiteContext?: 'none' | 'lax' | 'strict';
+ sort?: boolean;
+}
+
+/**
+ * Cookie jar contract, structurally compatible with `tough-cookie`'s `CookieJar` — which is
+ * what backs {@apilink Session} by default — so `@crawlee/types` does not have to depend on
+ * `tough-cookie` for it.
+ */
+export interface CookieJar {
+ setCookie(
+ cookie: string | SessionCookie,
+ url: string | URL,
+ options?: CookieJarSetCookieOptions,
+ ): Promise<SessionCookie | undefined>;
+ getCookies(url: string | URL, options?: CookieJarGetCookiesOptions): Promise<SessionCookie[]>;
+ getCookieString(url: string | URL, options?: CookieJarGetCookiesOptions): Promise<string>;
+ getSetCookieStrings(url: string | URL, options?: CookieJarGetCookiesOptions): Promise<string[] | undefined>;
+ serialize(): Promise<SerializedCookieJar>;
+ toJSON(): SerializedCookieJar | undefined;
+ clone(): Promise<CookieJar>;
+}
+
+/**
+ * JSON representation of a {@apilink CookieJar}. Structurally compatible with `tough-cookie`'s
+ * `SerializedCookieJar`.
+ */
+export interface SerializedCookieJar {
+ version: string;
+ storeType: string | null;
+ rejectPublicSuffixes: boolean;
+ cookies: Record<string, unknown>[];
+ [key: string]: unknown;
+}
diff --git a/packages/types/src/http-client.ts b/packages/types/src/http-client.ts
index 5362750f0..b7e6e0d2f 100644
--- a/packages/types/src/http-client.ts
+++ b/packages/types/src/http-client.ts
@@ -1,7 +1,6 @@
import type { Readable } from 'node:stream';
-import type { CookieJar } from 'tough-cookie';
-
+import type { CookieJar } from './cookies.js';
import type { ISession } from './session.js';
import type { AllowedHttpMethods } from './utility-types.js';
diff --git a/packages/types/src/index.ts b/packages/types/src/index.ts
index e4af087a5..c0da6dc91 100644
--- a/packages/types/src/index.ts
+++ b/packages/types/src/index.ts
@@ -2,6 +2,7 @@ export type * from './status-message.js';
export type * from './storages.js';
export type * from './utility-types.js';
export type * from './browser.js';
+export type * from './cookies.js';
export type * from './http-client.js';
export type * from './session.js';
export type * from './logger.js';
diff --git a/packages/types/src/session.ts b/packages/types/src/session.ts
index bae27cf24..70618eff6 100644
--- a/packages/types/src/session.ts
+++ b/packages/types/src/session.ts
@@ -1,4 +1,4 @@
-import type { CookieJar, SerializedCookieJar } from 'tough-cookie';
+import type { CookieJar, SerializedCookieJar } from './cookies.js';
/**
* The main purpose of the ProxyInfo object is to provide information
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index c5c5b6d95..4049d26ca 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -895,9 +895,6 @@ importers:
packages/types:
dependencies:
- tough-cookie:
- specifier: ^6.0.0
- version: 6.0.1
tslib:
specifier: ^2.8.1
version: 2.8.1
diff --git a/test/core/crawlers/browser_crawler.test.ts b/test/core/crawlers/browser_crawler.test.ts
index 9aa3460ca..462ba9bbb 100644
--- a/test/core/crawlers/browser_crawler.test.ts
+++ b/test/core/crawlers/browser_crawler.test.ts
@@ -644,8 +644,7 @@ describe('BrowserCrawler', () => {
requestList,
saveResponseCookies: true,
requestHandler: async ({ session, request }) => {
- loadedCookies.push(session.cookieJar.getCookieStringSync(request.url));
- return Promise.resolve();
+ loadedCookies.push(await session.cookieJar.getCookieString(request.url));
},
preNavigationHooks: [
async ({ session, page }) => {
@@ -696,7 +695,7 @@ describe('BrowserCrawler', () => {
maxPoolSize: 1,
}),
requestHandler: async ({ page, session, request }) => {
- cookieStrings.push(session.cookieJar.getCookieStringSync(request.url));
+ cookieStrings.push(await session.cookieJar.getCookieString(request.url));
if (request.url.includes('cookie-1')) {
const hostname = new URL(request.loadedUrl || request.url).hostname;
diff --git a/test/core/crawlers/cheerio_crawler.test.ts b/test/core/crawlers/cheerio_crawler.test.ts
index aaa733282..d019c7b29 100644
--- a/test/core/crawlers/cheerio_crawler.test.ts
+++ b/test/core/crawlers/cheerio_crawler.test.ts
@@ -1253,10 +1253,10 @@ describe('CheerioCrawler', () => {
maxPoolSize: 1,
}),
preNavigationHooks: [
- ({ session, request }) => {
+ async ({ session, request }) => {
// this should get overriden by the server
- session.cookieJar.setCookieSync('foo=bar1', request.url);
- session.cookieJar.setCookieSync('other=cookie1', request.url);
+ await session.cookieJar.setCookie('foo=bar1', request.url);
+ await session.cookieJar.setCookie('other=cookie1', request.url);
request.headers ??= {};
request.headers.cookie += '; coo=kie';
@@ -1505,7 +1505,7 @@ describe('CheerioCrawler', () => {
},
]),
requestHandler: async ({ session, request }) => {
- sessionCookies.push(session.cookieJar.getCookieStringSync(request.url));
+ sessionCookies.push(await session.cookieJar.getCookieString(request.url));
},
});
@@ -1522,8 +1522,8 @@ describe('CheerioCrawler', () => {
maxConcurrency: 1,
requestList: await RequestList.open(null, [`${serverAddress}/special/get-cookies`]),
preNavigationHooks: [
- ({ session, request }) => {
- session.cookieJar.setCookieSync('manual=fromHook', request.url);
+ async ({ session, request }) => {
+ await session.cookieJar.setCookie('manual=fromHook', request.url);
},
],
requestHandler: ({ json }) => {
diff --git a/test/core/crawlers/puppeteer_crawler.test.ts b/test/core/crawlers/puppeteer_crawler.test.ts
index 054c5f84c..a5344be85 100644
--- a/test/core/crawlers/puppeteer_crawler.test.ts
+++ b/test/core/crawlers/puppeteer_crawler.test.ts
@@ -319,15 +319,15 @@ describe('PuppeteerCrawler', () => {
saveResponseCookies: true,
sessionPool: new SessionPool({
- createSessionFunction: () => {
+ createSessionFunction: async () => {
const session = new Session();
- session.cookieJar.setCookieSync('example_cookie_name=example_cookie_value', serverUrl);
+ await session.cookieJar.setCookie('example_cookie_name=example_cookie_value', serverUrl);
return session;
},
}),
requestHandler: async ({ page, session }) => {
pageCookies = await page.cookies().then((cks) => cks.map((c) => `${c.name}=${c.value}`).join('; '));
- sessionCookies = session!.cookieJar.getCookieStringSync(serverUrl);
+ sessionCookies = await session!.cookieJar.getCookieString(serverUrl);
},
});
diff --git a/test/core/session_pool/session.test.ts b/test/core/session_pool/session.test.ts
index 98ddac123..1fc83a41b 100644
--- a/test/core/session_pool/session.test.ts
+++ b/test/core/session_pool/session.test.ts
@@ -143,9 +143,9 @@ describe('Session - testing session behaviour', () => {
expect(session.cookieJar.setCookie).toBeDefined();
});
- test('setCookie does not throw on malformed raw cookie string', () => {
+ test('setCookie does not throw on malformed raw cookie string', async () => {
session = new Session();
- expect(() => session.setCookie('garbled!!!@#$%nonsense', 'https://www.example.com')).not.toThrow();
+ await expect(session.setCookie('garbled!!!@#$%nonsense', 'https://www.example.com')).resolves.not.toThrow();
});
test('retired state survives a getState() / new Session() round-trip', () => {
@@ -168,11 +168,14 @@ describe('Session - testing session behaviour', () => {
expect(reinitialized.isUsable()).toBe(false);
});
- test('should correctly persist and init cookieJar', () => {
+ test('should correctly persist and init cookieJar', async () => {
const newSession = new Session();
const url = 'https://example.com';
- newSession.cookieJar.setCookieSync('CSRF=e8b667; Domain=example.com; Secure', url);
- newSession.cookieJar.setCookieSync('id=a3fWa; Expires=Wed, 21 Oct 2099 07:28:00 GMT; Domain=example.com', url);
+ await newSession.cookieJar.setCookie('CSRF=e8b667; Domain=example.com; Secure', url);
+ await newSession.cookieJar.setCookie(
+ 'id=a3fWa; Expires=Wed, 21 Oct 2099 07:28:00 GMT; Domain=example.com',
+ url,
+ );
const old = newSession.getState();
@@ -183,6 +186,6 @@ describe('Session - testing session behaviour', () => {
// @ts-expect-error string -> Date for createdAt has been overridden
const reinitializedSession = new Session({ ...old });
- expect(reinitializedSession.getCookieString(url)).toEqual('CSRF=e8b667; id=a3fWa');
+ await expect(reinitializedSession.getCookieString(url)).resolves.toEqual('CSRF=e8b667; id=a3fWa');
});
});
diff --git a/test/core/session_pool/session_pool.test.ts b/test/core/session_pool/session_pool.test.ts
index 0319255f4..7c1e489b7 100644
--- a/test/core/session_pool/session_pool.test.ts
+++ b/test/core/session_pool/session_pool.test.ts
@@ -114,8 +114,8 @@ describe('SessionPool - testing session pool', () => {
test('get state should work', async () => {
const url = 'https://example.com';
const newSession = await sessionPool.getSession();
- newSession?.cookieJar.setCookieSync('cookie1=my-cookie', url);
- newSession?.cookieJar.setCookieSync('cookie2=your-cookie', url);
+ await newSession?.cookieJar.setCookie('cookie1=my-cookie', url);
+ await newSession?.cookieJar.setCookie('cookie2=your-cookie', url);
const state = await sessionPool.getState();
expect(state).toBeInstanceOf(Object);