Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

feat: drop `gotOptions` params from the public interface (#3300)

Jindřich Bärf57ff3dd2eb6
diff --git a/docs/upgrading/upgrading_v4.md b/docs/upgrading/upgrading_v4.md
index 2a378d0e1..12a87d5db 100644
--- a/docs/upgrading/upgrading_v4.md
+++ b/docs/upgrading/upgrading_v4.md
@@ -110,3 +110,7 @@ The crawling context in the `FileDownload` crawler no longer includes the `body`
## `KeyValueStore.getPublicUrl` is now async
The `KeyValueStore.getPublicUrl` method is now asynchronous and reads the public URL directly from the storage client.
+
+## `preNavigationHooks` in `HttpCrawler` no longer accepts `gotOptions` object
+
+The `preNavigationHooks` option in `HttpCrawler` subclasses no longer accepts the `gotOptions` object as a second parameter. Modify the `crawlingContext` fields (e.g. `.request`) directly instead.
diff --git a/packages/cheerio-crawler/src/internals/cheerio-crawler.ts b/packages/cheerio-crawler/src/internals/cheerio-crawler.ts
index a7cc15046..abe9c6725 100644
--- a/packages/cheerio-crawler/src/internals/cheerio-crawler.ts
+++ b/packages/cheerio-crawler/src/internals/cheerio-crawler.ts
@@ -121,11 +121,11 @@ export type CheerioRequestHandler<
*
* The crawler finishes when there are no more {@apilink Request} objects to crawl.
*
- * We can use the `preNavigationHooks` to adjust `gotOptions`:
+ * We can use the `preNavigationHooks` to adjust the crawling context before the request is made:
*
* ```
* preNavigationHooks: [
- * (crawlingContext, gotOptions) => {
+ * (crawlingContext) => {
* // ...
* },
* ]
diff --git a/packages/core/src/http_clients/got-scraping-http-client.ts b/packages/core/src/http_clients/got-scraping-http-client.ts
index 98f206830..a248fcc13 100644
--- a/packages/core/src/http_clients/got-scraping-http-client.ts
+++ b/packages/core/src/http_clients/got-scraping-http-client.ts
@@ -38,7 +38,7 @@ export class GotScrapingHttpClient implements BaseHttpClient {
proxyUrl: session?.proxyInfo?.url,
timeout: { request: timeout },
// We set cookieJar to undefined because
- // `HttpCrawler` reads the cookies beforehand and sets them in `request.gotOptions`.
+ // `HttpCrawler` reads the cookies beforehand and sets them in `request.headers`.
// Using the `cookieJar` option directly would override that.
cookieJar: undefined,
});
diff --git a/packages/http-crawler/src/internals/file-download.ts b/packages/http-crawler/src/internals/file-download.ts
index 2151f485a..b06f3d71d 100644
--- a/packages/http-crawler/src/internals/file-download.ts
+++ b/packages/http-crawler/src/internals/file-download.ts
@@ -127,11 +127,11 @@ export function ByteCounterStream({
*
* The crawler finishes when there are no more {@apilink Request} objects to crawl.
*
- * We can use the `preNavigationHooks` to adjust `gotOptions`:
+ * We can use the `preNavigationHooks` to adjust the crawling context before the request is made:
*
* ```
* preNavigationHooks: [
- * (crawlingContext, gotOptions) => {
+ * (crawlingContext) => {
* // ...
* },
* ]
diff --git a/packages/http-crawler/src/internals/http-crawler.ts b/packages/http-crawler/src/internals/http-crawler.ts
index c5e3386ec..2801a1bd5 100644
--- a/packages/http-crawler/src/internals/http-crawler.ts
+++ b/packages/http-crawler/src/internals/http-crawler.ts
@@ -34,7 +34,7 @@ import { type CheerioRoot, RETRY_CSS_SELECTORS } from '@crawlee/utils';
import * as cheerio from 'cheerio';
import type { RequestLike, ResponseLike } from 'content-type';
import contentTypeParser from 'content-type';
-import type { Method, OptionsInit, TimeoutError as TimeoutErrorClass } from 'got-scraping';
+import type { TimeoutError as TimeoutErrorClass } from 'got-scraping';
import iconv from 'iconv-lite';
import ow from 'ow';
import type { JsonValue } from 'type-fest';
@@ -83,12 +83,12 @@ export interface HttpCrawlerOptions<
/**
* Async functions that are sequentially evaluated before the navigation. Good for setting additional cookies
- * or browser properties before navigation. The function accepts two parameters, `crawlingContext` and `gotOptions`,
- * which are passed to the `requestAsBrowser()` function the crawler calls to navigate.
+ * or browser properties before navigation. The function accepts one parameter `crawlingContext`,
+ * which is passed to the `requestAsBrowser()` function the crawler calls to navigate.
* Example:
* ```
* preNavigationHooks: [
- * async (crawlingContext, gotOptions) => {
+ * async (crawlingContext) => {
* // ...
* },
* ]
@@ -168,7 +168,7 @@ export interface HttpCrawlerOptions<
/**
* @internal
*/
-export type InternalHttpHook<Context> = (crawlingContext: Context, gotOptions: OptionsInit) => Awaitable<void>;
+export type InternalHttpHook<Context> = (crawlingContext: Context) => Awaitable<void>;
export type HttpHook<
UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
@@ -272,11 +272,11 @@ export type HttpRequestHandler<
*
* The crawler finishes when there are no more {@apilink Request} objects to crawl.
*
- * We can use the `preNavigationHooks` to adjust `gotOptions`:
+ * We can use the `preNavigationHooks` to adjust the crawling context before the request is made:
*
* ```javascript
* preNavigationHooks: [
- * (crawlingContext, gotOptions) => {
+ * (crawlingContext) => {
* // ...
* },
* ]
@@ -455,23 +455,22 @@ export class HttpCrawler<
};
}
- const gotOptions = {} as OptionsInit;
const preNavigationHooksCookies = this._getCookieHeaderFromRequest(request);
request.state = RequestState.BEFORE_NAV;
// Execute pre navigation hooks before applying session pool cookies,
// as they may also set cookies in the session
- await this._executeHooks(this.preNavigationHooks, crawlingContext, gotOptions);
+ await this._executeHooks(this.preNavigationHooks, crawlingContext);
tryCancel();
const postNavigationHooksCookies = this._getCookieHeaderFromRequest(request);
- this._applyCookies(crawlingContext, gotOptions, preNavigationHooksCookies, postNavigationHooksCookies);
+ const cookieString = this._applyCookies(crawlingContext, preNavigationHooksCookies, postNavigationHooksCookies);
const proxyUrl = crawlingContext.proxyInfo?.url;
const httpResponse = await addTimeoutToPromise(
- async () => this._requestFunction({ request, session, proxyUrl, gotOptions }),
+ async () => this._requestFunction({ request, session, proxyUrl, cookieString }),
this.navigationTimeoutMillis,
`request timed out after ${this.navigationTimeoutMillis / 1000} seconds.`,
);
@@ -588,60 +587,19 @@ export class HttpCrawler<
}
/**
- * Sets the cookie header to `gotOptions` based on the provided request and session headers, as well as any changes that occurred due to hooks.
+ * Returns the `Cookie` header value based on the current context and
+ * any changes that occurred in the navigation hooks.
*/
protected _applyCookies(
{ session, request }: CrawlingContext,
- gotOptions: OptionsInit,
preHookCookies: string,
postHookCookies: string,
- ) {
+ ): string {
const sessionCookie = session?.getCookieString(request.url) ?? '';
- let alteredGotOptionsCookies = gotOptions.headers?.Cookie || gotOptions.headers?.cookie || '';
-
- if (gotOptions.headers?.Cookie && gotOptions.headers?.cookie) {
- const { Cookie: upperCaseHeader, cookie: lowerCaseHeader } = gotOptions.headers;
-
- this.log.warning(
- `Encountered mixed casing for the cookie headers in the got options for request ${request.url} (${request.id}). Their values will be merged`,
- );
-
- const sourceCookies = [];
-
- if (Array.isArray(lowerCaseHeader)) {
- sourceCookies.push(...lowerCaseHeader);
- } else {
- sourceCookies.push(lowerCaseHeader);
- }
-
- if (Array.isArray(upperCaseHeader)) {
- sourceCookies.push(...upperCaseHeader);
- } else {
- sourceCookies.push(upperCaseHeader);
- }
- alteredGotOptionsCookies = mergeCookies(request.url, sourceCookies);
- }
-
- const sourceCookies = [sessionCookie, preHookCookies];
-
- if (Array.isArray(alteredGotOptionsCookies)) {
- sourceCookies.push(...alteredGotOptionsCookies);
- } else {
- sourceCookies.push(alteredGotOptionsCookies);
- }
-
- sourceCookies.push(postHookCookies);
-
- const mergedCookie = mergeCookies(request.url, sourceCookies);
+ const sourceCookies = [sessionCookie, preHookCookies, postHookCookies];
- gotOptions.headers ??= {};
- Reflect.deleteProperty(gotOptions.headers, 'Cookie');
- Reflect.deleteProperty(gotOptions.headers, 'cookie');
-
- if (mergedCookie !== '') {
- gotOptions.headers.Cookie = mergedCookie;
- }
+ return mergeCookies(request.url, sourceCookies);
}
/**
@@ -653,17 +611,17 @@ export class HttpCrawler<
request,
session,
proxyUrl,
- gotOptions,
+ cookieString,
}: RequestFunctionOptions): Promise<Response> {
if (!TimeoutError) {
// @ts-ignore
({ TimeoutError } = await import('got-scraping'));
}
- const opts = this._getRequestOptions(request, session, proxyUrl, gotOptions);
+ const opts = this._getRequestOptions(request, session, proxyUrl);
try {
- return await this._requestAsBrowser(opts, session);
+ return await this._requestAsBrowser(opts, session, cookieString);
} catch (e) {
if (e instanceof TimeoutError) {
this._handleRequestTimeout(session);
@@ -727,26 +685,19 @@ export class HttpCrawler<
/**
* Combines the provided `requestOptions` with mandatory (non-overridable) values.
*/
- protected _getRequestOptions(
- request: CrawleeRequest,
- session?: Session,
- proxyUrl?: string,
- gotOptions?: OptionsInit,
- ) {
- const requestOptions: OptionsInit & Required<Pick<OptionsInit, 'url'>> & { isStream: true } = {
+ protected _getRequestOptions(request: CrawleeRequest, session?: Session, proxyUrl?: string) {
+ const requestOptions = {
url: request.url,
- method: request.method as Method,
+ method: request.method,
proxyUrl,
timeout: { request: this.navigationTimeoutMillis },
cookieJar: this.persistCookiesPerSession ? session?.cookieJar : undefined,
sessionToken: session,
- ...gotOptions,
- headers: { ...request.headers, ...gotOptions?.headers },
+ headers: request.headers,
https: {
- ...gotOptions?.https,
rejectUnauthorized: !this.ignoreSslErrors,
},
- isStream: true,
+ body: undefined as string | undefined,
};
// Delete any possible lowercased header for cookie as they are merged in _applyCookies under the uppercase Cookie header
@@ -858,16 +809,19 @@ export class HttpCrawler<
/**
* @internal wraps public utility for mocking purposes
*/
- private _requestAsBrowser = async (
- options: OptionsInit & { url: string | URL; isStream: true },
- session?: Session,
- ) => {
+ private _requestAsBrowser = async (options: Dictionary<any>, session?: Session, cookieString?: string) => {
const opts = processHttpRequestOptions({
...(options as any),
cookieJar: options.cookieJar,
responseType: 'text',
});
+ if (cookieString) {
+ opts.headers?.delete('Cookie');
+ opts.headers?.delete('cookie');
+ opts.headers?.set('Cookie', cookieString);
+ }
+
const response = await this.httpClient.stream(
new Request(opts.url, {
body: opts.body ? (Readable.toWeb(opts.body) as any) : undefined,
@@ -883,9 +837,9 @@ export class HttpCrawler<
if (this.persistCookiesPerSession) {
session!.setCookiesFromResponse(redirectResponse);
- const cookieString = session!.getCookieString(updatedRequest.url!.toString());
- if (cookieString !== '') {
- updatedRequest.headers.set('Cookie', cookieString);
+ const cookieStringRedirected = session!.getCookieString(updatedRequest.url!.toString());
+ if (cookieStringRedirected !== '') {
+ updatedRequest.headers.set('Cookie', cookieStringRedirected);
}
}
},
@@ -900,7 +854,7 @@ interface RequestFunctionOptions {
request: CrawleeRequest;
session?: Session;
proxyUrl?: string;
- gotOptions: OptionsInit;
+ cookieString?: string;
}
/**
diff --git a/packages/jsdom-crawler/src/internals/jsdom-crawler.ts b/packages/jsdom-crawler/src/internals/jsdom-crawler.ts
index 7ef79c0fa..4bf2a049e 100644
--- a/packages/jsdom-crawler/src/internals/jsdom-crawler.ts
+++ b/packages/jsdom-crawler/src/internals/jsdom-crawler.ts
@@ -133,11 +133,11 @@ export type JSDOMRequestHandler<
*
* The crawler finishes when there are no more {@apilink Request} objects to crawl.
*
- * We can use the `preNavigationHooks` to adjust `gotOptions`:
+ * We can use the `preNavigationHooks` to adjust the crawling context before the request is made:
*
* ```
* preNavigationHooks: [
- * (crawlingContext, gotOptions) => {
+ * (crawlingContext) => {
* // ...
* },
* ]
diff --git a/packages/linkedom-crawler/src/internals/linkedom-crawler.ts b/packages/linkedom-crawler/src/internals/linkedom-crawler.ts
index e65b59c44..86e2b0e68 100644
--- a/packages/linkedom-crawler/src/internals/linkedom-crawler.ts
+++ b/packages/linkedom-crawler/src/internals/linkedom-crawler.ts
@@ -121,11 +121,11 @@ export type LinkeDOMRequestHandler<
*
* The crawler finishes when there are no more {@apilink Request} objects to crawl.
*
- * We can use the `preNavigationHooks` to adjust `gotOptions`:
+ * We can use the `preNavigationHooks` to adjust the crawling context before the request is made:
*
* ```
* preNavigationHooks: [
- * (crawlingContext, gotOptions) => {
+ * (crawlingContext) => {
* // ...
* },
* ]
diff --git a/test/core/crawlers/cheerio_crawler.test.ts b/test/core/crawlers/cheerio_crawler.test.ts
index 85be4da0b..aa0a8365e 100644
--- a/test/core/crawlers/cheerio_crawler.test.ts
+++ b/test/core/crawlers/cheerio_crawler.test.ts
@@ -1022,9 +1022,8 @@ describe('CheerioCrawler', () => {
});
});
- test('should merge cookies set in pre-nav hook with the session ones', async () => {
+ test('should merge request and session cookies', async () => {
const responses: unknown[] = [];
- const gotOptions: OptionsInit[] = [];
const crawler = new CheerioCrawler({
requestList: await RequestList.open(null, [
{
@@ -1040,11 +1039,6 @@ describe('CheerioCrawler', () => {
requestHandler: ({ json }) => {
responses.push(json);
},
- preNavigationHooks: [
- (_context, options) => {
- gotOptions.push(options);
- },
- ],
});
const sessSpy = vitest.spyOn(Session.prototype, 'getCookieString');
@@ -1056,12 +1050,6 @@ describe('CheerioCrawler', () => {
cookie: 'foo=bar2; other=cookie1; coo=kie; baz=123',
},
});
- expect(gotOptions).toHaveLength(1);
- expect(gotOptions[0]).toMatchObject({
- headers: {
- Cookie: 'foo=bar2; other=cookie1; coo=kie; baz=123', // header name normalized to `Cookie`
- },
- });
});
test('should work with cookies adjusted on `context.request` in pre-nav hook', async () => {