Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

fix: rectify transformRequestFunction precedence rules (#3372)

Richard Solarb6639404d96b
diff --git a/docs/upgrading/upgrading_v4.md b/docs/upgrading/upgrading_v4.md
index 12a87d5db..d4d65a0f7 100644
--- a/docs/upgrading/upgrading_v4.md
+++ b/docs/upgrading/upgrading_v4.md
@@ -114,3 +114,18 @@ The `KeyValueStore.getPublicUrl` method is now asynchronous and reads the public
## `preNavigationHooks` in `HttpCrawler` no longer accepts `gotOptions` object
The `preNavigationHooks` option in `HttpCrawler` subclasses no longer accepts the `gotOptions` object as a second parameter. Modify the `crawlingContext` fields (e.g. `.request`) directly instead.
+
+## `transformRequestFunction` precedence in `enqueueLinks`
+
+The `transformRequestFunction` callback in `enqueueLinks` now runs **after** URL pattern filtering (`globs`, `regexps`, `pseudoUrls`) instead of before. This means it has the highest priority and can overwrite any request options set by patterns or the global `label` option.
+
+The priority order is now (lowest to highest):
+1. Global `label` / `userData` options
+2. Pattern-specific options from `globs`, `regexps`, or `pseudoUrls` objects
+3. `transformRequestFunction`
+
+The `transformRequestFunction` callback receives a `RequestOptions` object and can return either:
+- The modified `RequestOptions` object
+- A new `RequestOptions` plain object
+- `'unchanged'` to keep the original options as-is
+- A falsy value or `'skip'` to exclude the request from the queue
diff --git a/packages/basic-crawler/src/internals/basic-crawler.ts b/packages/basic-crawler/src/internals/basic-crawler.ts
index 7e2302a28..be944e57b 100644
--- a/packages/basic-crawler/src/internals/basic-crawler.ts
+++ b/packages/basic-crawler/src/internals/basic-crawler.ts
@@ -1800,17 +1800,22 @@ export class BasicCrawler<
request: Request<Dictionary>,
requestQueue: RequestProvider,
): Promise<BatchAddRequestsResult> {
- const transformRequestFunctionWrapper: RequestTransform = (newRequest) => {
- newRequest.crawlDepth = request.crawlDepth + 1;
-
- if (this.maxCrawlDepth !== undefined && newRequest.crawlDepth > this.maxCrawlDepth) {
- newRequest.skippedReason = 'depth';
+ const transformRequestFunctionWrapper: RequestTransform = (requestOptions) => {
+ requestOptions.crawlDepth = request.crawlDepth + 1;
+
+ if (this.maxCrawlDepth !== undefined && requestOptions.crawlDepth! > this.maxCrawlDepth) {
+ // Setting `skippedReason` before returning `false` ensures that `reportSkippedRequests`
+ // reports `'depth'` as the reason (via `request.skippedReason ?? reason` fallback),
+ // rather than the generic `'transform'` reason.
+ requestOptions.skippedReason = 'depth';
return false;
}
// After injecting the crawlDepth, we call the user-provided transform function, if there is one.
// Its return value is passed through as is, so a falsy one still skips the request.
- return options.transformRequestFunction ? options.transformRequestFunction(newRequest) : newRequest;
+ return options.transformRequestFunction
+ ? options.transformRequestFunction(requestOptions)
+ : requestOptions;
};
const limit = this.calculateEnqueuedRequestLimit(options.limit);
diff --git a/packages/core/src/enqueue_links/enqueue_links.ts b/packages/core/src/enqueue_links/enqueue_links.ts
index 8a6b796f2..f483587e8 100644
--- a/packages/core/src/enqueue_links/enqueue_links.ts
+++ b/packages/core/src/enqueue_links/enqueue_links.ts
@@ -6,7 +6,8 @@ import type { SetRequired } from 'type-fest';
import log from '@apify/log';
-import type { Request, RequestOptions } from '../request.js';
+import type { RequestOptions } from '../request.js';
+import { Request } from '../request.js';
import type {
AddRequestsBatchedOptions,
AddRequestsBatchedResult,
@@ -23,12 +24,12 @@ import type {
UrlPatternObject,
} from './shared.js';
import {
+ applyRequestTransform,
constructGlobObjectsFromGlobs,
constructRegExpObjectsFromPseudoUrls,
constructRegExpObjectsFromRegExps,
createRequestOptions,
- createRequests,
- filterRequestsByPatterns,
+ filterRequestOptionsByPatterns,
} from './shared.js';
export { EnqueueStrategy };
@@ -56,8 +57,8 @@ export interface EnqueueLinksOptions extends RequestQueueOperationOptions {
/**
* Sets {@apilink Request.label} for newly enqueued requests.
*
- * Note that the request options specified in `globs`, `regexps`, or `pseudoUrls` objects
- * have priority over this option.
+ * This option has the lowest priority and can be overwritten by request options
+ * specified in `globs`, `regexps`, or `pseudoUrls` objects, as well as by `transformRequestFunction`.
*/
label?: string;
@@ -132,12 +133,12 @@ export interface EnqueueLinksOptions extends RequestQueueOperationOptions {
pseudoUrls?: readonly PseudoUrlInput[];
/**
- * Just before a new {@apilink Request} is constructed and enqueued to the {@apilink RequestQueue}, this function can be used
- * to remove it or modify its contents such as `userData`, `payload` or, most importantly `uniqueKey`. This is useful
+ * After request options are filtered by patterns, this function can be used
+ * to remove them or modify their contents such as `userData`, `payload` or, most importantly `uniqueKey`. This is useful
* when you need to enqueue multiple `Requests` to the queue that share the same URL, but differ in methods or payloads,
* or to dynamically update or create `userData`.
*
- * For example: by adding `keepUrlFragment: true` to the `request` object, URL fragments will not be removed
+ * For example: by adding `keepUrlFragment: true` to the request options, URL fragments will not be removed
* when `uniqueKey` is computed.
*
* **Example:**
@@ -151,8 +152,13 @@ export interface EnqueueLinksOptions extends RequestQueueOperationOptions {
* }
* ```
*
- * Note that the request options specified in `globs`, `regexps`, or `pseudoUrls` objects
- * have priority over this function. Some request options returned by `transformRequestFunction` may be overwritten by pattern-based options from `globs`, `regexps`, or `pseudoUrls`.
+ * Note that `transformRequestFunction` has the highest priority and can overwrite request options
+ * specified in `globs`, `regexps`, or `pseudoUrls` objects, as well as the global `label` option.
+ *
+ * The function receives a {@apilink RequestOptions} object and can return either:
+ * - The modified {@apilink RequestOptions} object
+ * - `'unchanged'` to keep the original options as-is
+ * - A falsy value or `'skip'` to exclude the request from the queue
*/
transformRequestFunction?: RequestTransform;
@@ -401,58 +407,65 @@ export async function enqueueLinks(
await reportSkippedRequests(skippedRequests, 'robotsTxt');
}
- if (transformRequestFunction) {
- const skippedRequests: RequestOptions[] = [];
-
- requestOptions = requestOptions
- .map((request) => {
- const transformedRequest = transformRequestFunction(request);
- if (!transformedRequest) {
- skippedRequests.push(request);
- }
- return transformedRequest;
- })
- .filter((r) => Boolean(r)) as RequestOptions[];
-
- await reportSkippedRequests(skippedRequests, 'filters');
- }
-
async function createFilteredRequests() {
const skippedRequests: string[] = [];
- // No user provided patterns means we can skip an extra filtering step
+ // Step 1: Filter request options by exclude patterns, user patterns (globs/regexps), and strategy patterns.
+ // Pattern-level options (label, userData, method, etc.) are merged during this step.
+ let filteredOptions: RequestOptions[];
if (urlPatternObjects.length === 0) {
- return createRequests(
+ filteredOptions = filterRequestOptionsByPatterns(
+ requestOptions,
+ enqueueStrategyPatterns.length > 0 ? enqueueStrategyPatterns : undefined,
+ urlExcludePatternObjects,
+ options.strategy,
+ (url) => skippedRequests.push(url),
+ );
+ } else {
+ // Filter by user patterns first (with exclude)
+ const afterUserPatterns = filterRequestOptionsByPatterns(
requestOptions,
- enqueueStrategyPatterns,
+ urlPatternObjects,
urlExcludePatternObjects,
options.strategy,
(url) => skippedRequests.push(url),
);
+ // ...then filter by the enqueue links strategy (making this an AND check)
+ filteredOptions = filterRequestOptionsByPatterns(
+ afterUserPatterns,
+ enqueueStrategyPatterns.length > 0 ? enqueueStrategyPatterns : undefined,
+ [],
+ options.strategy,
+ (url) => skippedRequests.push(url),
+ );
}
- // Generate requests based on the user patterns first
- const generatedRequestsFromUserFilters = createRequests(
- requestOptions,
- urlPatternObjects,
- urlExcludePatternObjects,
- options.strategy,
- (url) => skippedRequests.push(url),
- );
- // ...then filter them by the enqueue links strategy (making this an AND check)
- const filtered = filterRequestsByPatterns(generatedRequestsFromUserFilters, enqueueStrategyPatterns, (url) =>
- skippedRequests.push(url),
- );
-
await reportSkippedRequests(
skippedRequests.map((url) => ({ url })),
'filters',
);
- return filtered;
+ // Step 2: Apply transformRequestFunction on request options - it has the highest priority
+ if (transformRequestFunction) {
+ const skippedByTransform: RequestOptions[] = [];
+ filteredOptions = applyRequestTransform(filteredOptions, transformRequestFunction, (r) =>
+ skippedByTransform.push(r),
+ );
+ await reportSkippedRequests(skippedByTransform, 'transform');
+ }
+
+ // Step 3: Create Request instances from the final request options
+ return filteredOptions.map((opts) => new Request(opts));
+ }
+
+ let requests = await createFilteredRequests();
+
+ if (typeof limit === 'number' && limit < requests.length) {
+ await reportSkippedRequests(requests.slice(limit), 'enqueueLimit');
+ requests = requests.slice(0, limit);
}
- const { addedRequests, requestsOverLimit } = await requestQueue.addRequestsBatched(await createFilteredRequests(), {
+ const { addedRequests, requestsOverLimit } = await requestQueue.addRequestsBatched(requests, {
forefront,
waitForAllRequestsToBeAdded,
maxNewRequests: limit,
diff --git a/packages/core/src/enqueue_links/shared.ts b/packages/core/src/enqueue_links/shared.ts
index 86a0bbd8b..99ed092ed 100644
--- a/packages/core/src/enqueue_links/shared.ts
+++ b/packages/core/src/enqueue_links/shared.ts
@@ -6,7 +6,6 @@ import { Minimatch } from 'minimatch';
import { purlToRegExp } from '@apify/pseudo_url';
import type { RequestOptions } from '../request.js';
-import { Request } from '../request.js';
import type { EnqueueLinksOptions } from './enqueue_links.js';
export { tryAbsoluteURL } from '@crawlee/utils';
@@ -47,7 +46,14 @@ export type RegExpObject = { regexp: RegExp } & Pick<
export type RegExpInput = RegExp | RegExpObject;
-export type SkippedRequestReason = 'robotsTxt' | 'limit' | 'enqueueLimit' | 'filters' | 'redirect' | 'depth';
+export type SkippedRequestReason =
+ | 'robotsTxt'
+ | 'limit'
+ | 'enqueueLimit'
+ | 'filters'
+ | 'transform'
+ | 'redirect'
+ | 'depth';
export type SkippedRequestCallback = (args: { url: string; reason: SkippedRequestReason }) => Awaitable<void>;
@@ -164,76 +170,46 @@ export function constructRegExpObjectsFromRegExps(regexps: readonly RegExpInput[
}
/**
+ * Filters request options by URL patterns and merges pattern-level options (label, userData, method, payload, headers)
+ * from the first matching pattern into each RequestOptions entry.
+ *
+ * When `includePatterns` is empty/undefined, all options pass through (only exclude filtering applies).
* @ignore
*/
-export function createRequests(
- requestOptions: (string | RequestOptions)[],
- urlPatternObjects?: UrlPatternObject[],
- excludePatternObjects: UrlPatternObject[] = [],
+export function filterRequestOptionsByPatterns(
+ requestOptions: RequestOptions[],
+ includePatterns: UrlPatternObject[] | undefined,
+ excludePatterns: UrlPatternObject[] = [],
strategy?: EnqueueLinksOptions['strategy'],
onSkippedUrl?: (url: string) => void,
-): Request[] {
- const excludePatternObjectMatchers = excludePatternObjects.map(createPatternObjectMatcher);
- const urlPatternObjectMatchers = urlPatternObjects?.map(createPatternObjectMatcher);
+): RequestOptions[] {
+ const excludeMatchers = excludePatterns.map(createPatternObjectMatcher);
+ const includeMatchers = includePatterns?.length ? includePatterns.map(createPatternObjectMatcher) : undefined;
return requestOptions
- .map((opts) => ({ url: typeof opts === 'string' ? opts : opts.url, opts }))
.filter(({ url }) => {
- const matchesExcludePatterns = excludePatternObjectMatchers.some(({ match }) => match(url));
-
- if (matchesExcludePatterns) {
+ const matchesExclude = excludeMatchers.some(({ match }) => match(url));
+ if (matchesExclude) {
onSkippedUrl?.(url);
}
-
- return !matchesExcludePatterns;
+ return !matchesExclude;
})
- .map(({ url, opts }) => {
- if (!urlPatternObjectMatchers || !urlPatternObjectMatchers.length) {
- return new Request(typeof opts === 'string' ? { url: opts, enqueueStrategy: strategy } : { ...opts });
+ .map((opts) => {
+ if (!includeMatchers) {
+ return { ...opts, enqueueStrategy: strategy };
}
- for (const urlPatternObject of urlPatternObjectMatchers) {
- const { match, glob, regexp, ...requestRegExpOptions } = urlPatternObject;
- if (match(url)) {
- const request =
- typeof opts === 'string'
- ? { url: opts, ...requestRegExpOptions, enqueueStrategy: strategy }
- : { ...opts, ...requestRegExpOptions, enqueueStrategy: strategy };
-
- return new Request(request);
+ for (const { match, glob, regexp, ...patternOptions } of includeMatchers) {
+ if (match(opts.url)) {
+ return { ...opts, ...patternOptions, enqueueStrategy: strategy };
}
}
// didn't match any positive pattern
- onSkippedUrl?.(url);
+ onSkippedUrl?.(opts.url);
return null;
})
- .filter((request) => request) as Request[];
-}
-
-export function filterRequestsByPatterns(
- requests: Request[],
- patterns?: UrlPatternObject[],
- onSkippedUrl?: (url: string) => void,
-): Request[] {
- if (!patterns?.length) {
- return requests;
- }
-
- const filtered: Request[] = [];
- const patternMatchers = patterns?.map(createPatternObjectMatcher);
-
- for (const request of requests) {
- const matchingPattern = patternMatchers.find(({ match }) => match(request.url));
-
- if (matchingPattern !== undefined) {
- filtered.push(request);
- } else {
- onSkippedUrl?.(request.url);
- }
- }
-
- return filtered;
+ .filter((opts) => opts !== null);
}
/**
@@ -293,13 +269,45 @@ function createPatternObjectMatcher(urlPatternObject: UrlPatternObject) {
}
/**
- * Takes an Apify {@apilink RequestOptions} object and changes its attributes in a desired way. This user-function is used
- * {@apilink enqueueLinks} to modify requests before enqueuing them.
+ * Takes a {@apilink RequestOptions} object and changes its attributes in a desired way. This user-function is used
+ * by {@apilink enqueueLinks} to modify request options before they are converted to {@apilink Request} instances.
*/
export interface RequestTransform {
/**
* @param original Request options to be modified.
- * @returns The modified request options to enqueue, or any falsy value to skip the request.
+ * @returns The modified request options to enqueue, `'unchanged'` to keep the original options as-is,
+ * or a falsy value / `'skip'` to exclude the request from the queue.
*/
- (original: RequestOptions): RequestOptions | false | undefined | null;
+ (original: RequestOptions): RequestOptions | false | undefined | null | 'skip' | 'unchanged';
+}
+
+/**
+ * Applies a {@apilink RequestTransform} function to a list of request options.
+ * Options for which the transform returns a falsy value are removed from the list.
+ * @param onSkipped Called with the original request options when the transform returns a falsy value (i.e. the request is skipped).
+ * @ignore
+ * @internal
+ */
+export function applyRequestTransform(
+ requestOptions: RequestOptions[],
+ transformFn: RequestTransform,
+ onSkipped?: (requestOptions: RequestOptions) => void,
+): RequestOptions[] {
+ return requestOptions
+ .map((opts) => {
+ const transformed = transformFn(opts);
+ if (transformed === 'skip') {
+ onSkipped?.(opts);
+ return null;
+ }
+ if (transformed === 'unchanged') {
+ return opts;
+ }
+ if (!transformed) {
+ onSkipped?.(opts);
+ return null;
+ }
+ return transformed;
+ })
+ .filter((r): r is RequestOptions => r !== null);
}
diff --git a/packages/core/src/request.ts b/packages/core/src/request.ts
index 74acf076c..0238bc2a5 100644
--- a/packages/core/src/request.ts
+++ b/packages/core/src/request.ts
@@ -360,6 +360,24 @@ class CrawleeRequest<UserData extends Dictionary = Dictionary> {
}
}
+ /**
+ * Reason for skipping this request.
+ */
+ get skippedReason(): SkippedRequestReason | undefined {
+ return this.userData.__crawlee?.skippedReason;
+ }
+
+ /**
+ * Reason for skipping this request.
+ */
+ set skippedReason(value: SkippedRequestReason | undefined) {
+ if (!this.userData.__crawlee) {
+ (this.userData as Dictionary).__crawlee = { skippedReason: value };
+ } else {
+ this.userData.__crawlee.skippedReason = value;
+ }
+ }
+
private get enqueueStrategy(): EnqueueLinksOptions['strategy'] | undefined {
return this.userData.__crawlee?.enqueueStrategy;
}
diff --git a/packages/playwright-crawler/src/internals/enqueue-links/click-elements.ts b/packages/playwright-crawler/src/internals/enqueue-links/click-elements.ts
index 33fd724c0..d83c0fd71 100644
--- a/packages/playwright-crawler/src/internals/enqueue-links/click-elements.ts
+++ b/packages/playwright-crawler/src/internals/enqueue-links/click-elements.ts
@@ -7,14 +7,17 @@ import type {
RequestOptions,
RequestProvider,
RequestTransform,
+ SkippedRequestCallback,
UrlPatternObject,
} from '@crawlee/browser';
import {
+ applyRequestTransform,
constructGlobObjectsFromGlobs,
constructRegExpObjectsFromPseudoUrls,
constructRegExpObjectsFromRegExps,
createRequestOptions,
- createRequests,
+ filterRequestOptionsByPatterns,
+ Request as CrawleeRequest,
} from '@crawlee/browser';
import type { BatchAddRequestsResult, Dictionary } from '@crawlee/types';
import ow from 'ow';
@@ -117,25 +120,28 @@ export interface EnqueueLinksByClickingElementsOptions {
pseudoUrls?: PseudoUrlInput[];
/**
- * Just before a new {@apilink Request} is constructed and enqueued to the {@apilink RequestQueue}, this function can be used
- * to remove it or modify its contents such as `userData`, `payload` or, most importantly `uniqueKey`. This is useful
- * when you need to enqueue multiple `Requests` to the queue that share the same URL, but differ in methods or payloads,
- * or to dynamically update or create `userData`.
- *
- * For example: by adding `useExtendedUniqueKey: true` to the `request` object, `uniqueKey` will be computed from
- * a combination of `url`, `method` and `payload` which enables crawling of websites that navigate using form submits
- * (POST requests).
+ * After {@apilink Request} objects are constructed and filtered by URL patterns (`globs`, `regexps`, `pseudoUrls`),
+ * this function can be used to remove them or modify their contents such as `userData`, `payload` or, most importantly
+ * `uniqueKey`. This is useful when you need to enqueue multiple `Requests` to the queue that share the same URL,
+ * but differ in methods or payloads, or to dynamically update or create `userData`.
*
* **Example:**
* ```javascript
* {
* transformRequestFunction: (request) => {
* request.userData.foo = 'bar';
- * request.useExtendedUniqueKey = true;
* return request;
* }
* }
* ```
+ *
+ * Note that `transformRequestFunction` has the highest priority and can overwrite request options
+ * specified in `globs`, `regexps`, or `pseudoUrls` objects, as well as the global `label` option.
+ *
+ * The function receives a {@apilink RequestOptions} object and can return either:
+ * - The modified {@apilink RequestOptions} object
+ * - `'unchanged'` to keep the original options as-is
+ * - A falsy value or `'skip'` to exclude the request from the queue
*/
transformRequestFunction?: RequestTransform;
@@ -179,6 +185,13 @@ export interface EnqueueLinksByClickingElementsOptions {
* @default false
*/
skipNavigation?: boolean;
+
+ /**
+ * When a request is skipped for some reason, you can use this callback to act on it.
+ * This is fired for requests skipped because they don't match enqueueLinks filters
+ * or because they were removed by `transformRequestFunction`.
+ */
+ onSkippedRequest?: SkippedRequestCallback;
}
/**
@@ -245,6 +258,7 @@ export async function enqueueLinksByClickingElements(
label: ow.optional.string,
forefront: ow.optional.boolean,
skipNavigation: ow.optional.boolean,
+ onSkippedRequest: ow.optional.function,
}),
);
@@ -261,6 +275,7 @@ export async function enqueueLinksByClickingElements(
maxWaitForPageIdleSecs = 5,
forefront,
exclude,
+ onSkippedRequest,
} = options;
const waitForPageIdleMillis = waitForPageIdleSecs * 1000;
@@ -299,11 +314,31 @@ export async function enqueueLinksByClickingElements(
maxWaitForPageIdleMillis,
clickOptions,
});
- let requestOptions = createRequestOptions(interceptedRequests, options);
+ const requestOptions = createRequestOptions(interceptedRequests, options);
+ const skippedByFilters: string[] = [];
+ let filteredOptions = filterRequestOptionsByPatterns(
+ requestOptions,
+ urlPatternObjects.length > 0 ? urlPatternObjects : undefined,
+ urlExcludePatternObjects,
+ undefined,
+ (url) => skippedByFilters.push(url),
+ );
+
+ if (onSkippedRequest && skippedByFilters.length > 0) {
+ await Promise.all(skippedByFilters.map((url) => onSkippedRequest({ url, reason: 'filters' })));
+ }
+
if (transformRequestFunction) {
- requestOptions = requestOptions.map(transformRequestFunction).filter((r) => !!r) as RequestOptions[];
+ const skippedByTransform: RequestOptions[] = [];
+ filteredOptions = applyRequestTransform(filteredOptions, transformRequestFunction, (r) =>
+ skippedByTransform.push(r),
+ );
+ if (onSkippedRequest && skippedByTransform.length > 0) {
+ await Promise.all(skippedByTransform.map((r) => onSkippedRequest({ url: r.url, reason: 'transform' })));
+ }
}
- const requests = createRequests(requestOptions, urlPatternObjects, urlExcludePatternObjects);
+
+ const requests = filteredOptions.map((opts) => new CrawleeRequest(opts));
const { addedRequests } = await requestQueue.addRequestsBatched(requests, { forefront });
return { processedRequests: addedRequests, unprocessedRequests: [] };
diff --git a/packages/puppeteer-crawler/src/internals/enqueue-links/click-elements.ts b/packages/puppeteer-crawler/src/internals/enqueue-links/click-elements.ts
index 0d67e0852..b73203ab1 100644
--- a/packages/puppeteer-crawler/src/internals/enqueue-links/click-elements.ts
+++ b/packages/puppeteer-crawler/src/internals/enqueue-links/click-elements.ts
@@ -7,14 +7,17 @@ import type {
RequestOptions,
RequestProvider,
RequestTransform,
+ SkippedRequestCallback,
UrlPatternObject,
} from '@crawlee/browser';
import {
+ applyRequestTransform,
constructGlobObjectsFromGlobs,
constructRegExpObjectsFromPseudoUrls,
constructRegExpObjectsFromRegExps,
createRequestOptions,
- createRequests,
+ filterRequestOptionsByPatterns,
+ Request,
} from '@crawlee/browser';
import type { BatchAddRequestsResult, Dictionary } from '@crawlee/types';
import ow from 'ow';
@@ -118,25 +121,28 @@ export interface EnqueueLinksByClickingElementsOptions {
pseudoUrls?: PseudoUrlInput[];
/**
- * Just before a new {@apilink Request} is constructed and enqueued to the {@apilink RequestQueue}, this function can be used
- * to remove it or modify its contents such as `userData`, `payload` or, most importantly `uniqueKey`. This is useful
- * when you need to enqueue multiple `Requests` to the queue that share the same URL, but differ in methods or payloads,
- * or to dynamically update or create `userData`.
- *
- * For example: by adding `useExtendedUniqueKey: true` to the `request` object, `uniqueKey` will be computed from
- * a combination of `url`, `method` and `payload` which enables crawling of websites that navigate using form submits
- * (POST requests).
+ * After {@apilink Request} objects are constructed and filtered by URL patterns (`globs`, `regexps`, `pseudoUrls`),
+ * this function can be used to remove them or modify their contents such as `userData`, `payload` or, most importantly
+ * `uniqueKey`. This is useful when you need to enqueue multiple `Requests` to the queue that share the same URL,
+ * but differ in methods or payloads, or to dynamically update or create `userData`.
*
* **Example:**
* ```javascript
* {
* transformRequestFunction: (request) => {
* request.userData.foo = 'bar';
- * request.useExtendedUniqueKey = true;
* return request;
* }
* }
* ```
+ *
+ * Note that `transformRequestFunction` has the highest priority and can overwrite request options
+ * specified in `globs`, `regexps`, or `pseudoUrls` objects, as well as the global `label` option.
+ *
+ * The function receives a {@apilink RequestOptions} object and can return either:
+ * - The modified {@apilink RequestOptions} object
+ * - `'unchanged'` to keep the original options as-is
+ * - A falsy value or `'skip'` to exclude the request from the queue
*/
transformRequestFunction?: RequestTransform;
@@ -180,6 +186,13 @@ export interface EnqueueLinksByClickingElementsOptions {
* @default false
*/
skipNavigation?: boolean;
+
+ /**
+ * When a request is skipped for some reason, you can use this callback to act on it.
+ * This is fired for requests skipped because they don't match enqueueLinks filters
+ * or because they were removed by `transformRequestFunction`.
+ */
+ onSkippedRequest?: SkippedRequestCallback;
}
/**
@@ -246,6 +259,7 @@ export async function enqueueLinksByClickingElements(
label: ow.optional.string,
forefront: ow.optional.boolean,
skipNavigation: ow.optional.boolean,
+ onSkippedRequest: ow.optional.function,
}),
);
@@ -262,6 +276,7 @@ export async function enqueueLinksByClickingElements(
maxWaitForPageIdleSecs = 5,
forefront,
exclude,
+ onSkippedRequest,
} = options;
const waitForPageIdleMillis = waitForPageIdleSecs * 1000;
@@ -300,11 +315,30 @@ export async function enqueueLinksByClickingElements(
maxWaitForPageIdleMillis,
clickOptions,
});
- let requestOptions = createRequestOptions(interceptedRequests, options);
+ const requestOptions = createRequestOptions(interceptedRequests, options);
+ const skippedByFilters: string[] = [];
+ let filteredOptions = filterRequestOptionsByPatterns(
+ requestOptions,
+ urlPatternObjects.length > 0 ? urlPatternObjects : undefined,
+ urlExcludePatternObjects,
+ undefined,
+ (url) => skippedByFilters.push(url),
+ );
+
+ if (onSkippedRequest && skippedByFilters.length > 0) {
+ await Promise.all(skippedByFilters.map((url) => onSkippedRequest({ url, reason: 'filters' })));
+ }
+
if (transformRequestFunction) {
- requestOptions = requestOptions.map(transformRequestFunction).filter((r) => !!r) as RequestOptions[];
+ const skippedByTransform: RequestOptions[] = [];
+ filteredOptions = applyRequestTransform(filteredOptions, transformRequestFunction, (r) =>
+ skippedByTransform.push(r),
+ );
+ if (onSkippedRequest && skippedByTransform.length > 0) {
+ await Promise.all(skippedByTransform.map((r) => onSkippedRequest({ url: r.url, reason: 'transform' })));
+ }
}
- const requests = createRequests(requestOptions, urlPatternObjects, urlExcludePatternObjects);
+ const requests = filteredOptions.map((opts) => new Request(opts));
const { addedRequests } = await requestQueue.addRequestsBatched(requests, { forefront });
return { processedRequests: addedRequests, unprocessedRequests: [] };
diff --git a/test/core/crawlers/basic_crawler.test.ts b/test/core/crawlers/basic_crawler.test.ts
index 6cc22096d..bb90b707b 100644
--- a/test/core/crawlers/basic_crawler.test.ts
+++ b/test/core/crawlers/basic_crawler.test.ts
@@ -3,14 +3,7 @@ import type { Server } from 'node:http';
import http from 'node:http';
import type { AddressInfo } from 'node:net';
-import type {
- CrawlingContext,
- EnqueueLinksOptions,
- ErrorHandler,
- RequestHandler,
- RequestOptions,
- Source,
-} from '@crawlee/basic';
+import type { EnqueueLinksOptions, ErrorHandler, RequestHandler, RequestOptions, Source } from '@crawlee/basic';
import {
BasicCrawler,
Configuration,
@@ -326,6 +319,26 @@ describe('BasicCrawler', () => {
expect(skippedRequests[1]).toStrictEqual({ url: 'https://example.com/2/', reason: 'filters' });
},
);
+
+ it('should report depth-limited requests with reason "depth" even when user transformRequestFunction is provided', async () => {
+ const transformRequestFunction = vi.fn((req: RequestOptions) => req);
+ const requestWithMaxDepth = new Request({ url: 'https://example.com/', crawlDepth: 3 });
+ const optionsWithTransform = { ...options, transformRequestFunction };
+
+ await crawler.exposedEnqueueLinksWithCrawlDepth(optionsWithTransform, requestWithMaxDepth, requestQueue);
+
+ const requests = addRequestsBatchedMock.mock.calls[0][0];
+ expect(requests).toHaveLength(0);
+
+ // Depth-limited requests should not reach the user's transformRequestFunction
+ expect(transformRequestFunction).not.toHaveBeenCalled();
+
+ // The skipped reason should be 'depth', not 'transform'
+ const skippedRequests = onSkippedRequestMock.mock.calls.map((call) => call[0]);
+ expect(skippedRequests).toHaveLength(2);
+ expect(skippedRequests[0]).toStrictEqual({ url: 'https://example.com/1/', reason: 'depth' });
+ expect(skippedRequests[1]).toStrictEqual({ url: 'https://example.com/2/', reason: 'depth' });
+ });
});
it('addCrawlDepthRequestGenerator() should generate requests with maxCrawlDepth', async () => {
diff --git a/test/core/enqueue_links/enqueue_links.test.ts b/test/core/enqueue_links/enqueue_links.test.ts
index 820b20f0d..9dce392ff 100644
--- a/test/core/enqueue_links/enqueue_links.test.ts
+++ b/test/core/enqueue_links/enqueue_links.test.ts
@@ -7,6 +7,7 @@ import {
Configuration,
EnqueueStrategy,
launchPuppeteer,
+ Request,
RequestQueue,
} from '@crawlee/puppeteer';
import { type CheerioRoot, RobotsTxtFile } from '@crawlee/utils';
@@ -1079,6 +1080,271 @@ describe('enqueueLinks()', () => {
expect(enqueued[i].options!.waitForAllRequestsToBeAdded).toBe(true);
}
});
+
+ describe('label precedence', () => {
+ test('global label option is applied if no other label is provided', async () => {
+ const { enqueued, requestQueue } = createRequestQueueMock();
+
+ await cheerioCrawlerEnqueueLinks({
+ options: {
+ selector: '.click',
+ label: 'global-label',
+ globs: ['https://example.com/**/*'],
+ },
+ $,
+ requestQueue,
+ originalRequestUrl: 'https://example.com',
+ });
+
+ expect(enqueued).toHaveLength(2);
+ // Global label should be applied when no pattern-specific label is set
+ expect(enqueued[0].userData).toEqual({ label: 'global-label' });
+ expect(enqueued[1].userData).toEqual({ label: 'global-label' });
+ });
+
+ test('pattern label overrides global label', async () => {
+ const { enqueued, requestQueue } = createRequestQueueMock();
+
+ await cheerioCrawlerEnqueueLinks({
+ options: {
+ selector: '.click',
+ label: 'global-label',
+ regexps: [
+ { regexp: /example\.com\/a\/b\/first/, label: 'pattern-label' },
+ /example\.com\/a\/b\/third/, // No label, should use global
+ ],
+ },
+ $,
+ requestQueue,
+ originalRequestUrl: 'https://example.com',
+ });
+
+ expect(enqueued).toHaveLength(2);
+ // Pattern-specific label should override global label
+ expect(enqueued[0].url).toBe('https://example.com/a/b/first');
+ expect(enqueued[0].userData).toEqual({ label: 'pattern-label' });
+ // URL matching pattern without label should use global label
+ expect(enqueued[1].url).toBe('https://example.com/a/b/third');
+ expect(enqueued[1].userData).toEqual({ label: 'global-label' });
+ });
+
+ test('transformRequestFunction has highest priority and overrides pattern label', async () => {
+ const { enqueued, requestQueue } = createRequestQueueMock();
+
+ await cheerioCrawlerEnqueueLinks({
+ options: {
+ selector: '.click',
+ label: 'global-label',
+ regexps: [{ regexp: /example\.com/, label: 'pattern-label' }],
+ transformRequestFunction: (request) => {
+ if (request.url.includes('/a/b/first')) {
+ request.label = 'transformed-label';
+ }
+ return request;
+ },
+ },
+ $,
+ requestQueue,
+ originalRequestUrl: 'https://example.com',
+ });
+
+ expect(enqueued).toHaveLength(2);
+ // transformRequestFunction should override pattern label
+ expect(enqueued[0].url).toBe('https://example.com/a/b/first');
+ expect(enqueued[0].userData).toEqual({ label: 'transformed-label' });
+ // URL not modified by transformRequestFunction should keep pattern label
+ expect(enqueued[1].url).toBe('https://example.com/a/b/third');
+ expect(enqueued[1].userData).toEqual({ label: 'pattern-label' });
+ });
+
+ test('transformRequestFunction can override all label sources', async () => {
+ const { enqueued, requestQueue } = createRequestQueueMock();
+
+ await cheerioCrawlerEnqueueLinks({
+ options: {
+ selector: '.click',
+ label: 'global-label',
+ globs: [
+ { glob: 'https://example.com/a/b/first', label: 'glob-label' },
+ { glob: 'https://example.com/a/b/third', label: 'glob-label' },
+ { glob: 'http://cool.com/', label: 'cool-label' },
+ ],
+ transformRequestFunction: (request) => {
+ // Override all labels
+ request.label = 'final-label';
+ return request;
+ },
+ },
+ $,
+ requestQueue,
+ originalRequestUrl: 'https://example.com',
+ });
+
+ expect(enqueued).toHaveLength(3);
+ // All requests should have the transformed label
+ for (const request of enqueued) {
+ expect(request.userData).toEqual({ label: 'final-label' });
+ }
+ });
+
+ test('transformRequestFunction can modify other request properties after patterns are applied', async () => {
+ const { enqueued, requestQueue } = createRequestQueueMock();
+
+ await cheerioCrawlerEnqueueLinks({
+ options: {
+ selector: '.click',
+ regexps: [{ regexp: /example\.com/, method: 'POST' as const, userData: { source: 'pattern' } }],
+ transformRequestFunction: (request) => {
+ // Change method set by pattern
+ request.method = 'PUT';
+ // Add to userData without removing pattern's data
+ request.userData = { ...request.userData, transformed: true };
+ return request;
+ },
+ },
+ $,
+ requestQueue,
+ originalRequestUrl: 'https://example.com',
+ });
+
+ expect(enqueued).toHaveLength(2);
+ // transformRequestFunction should override method from pattern
+ expect(enqueued[0].method).toBe('PUT');
+ expect(enqueued[1].method).toBe('PUT');
+ // userData should contain both pattern and transformed data
+ expect(enqueued[0].userData).toEqual({ source: 'pattern', transformed: true });
+ expect(enqueued[1].userData).toEqual({ source: 'pattern', transformed: true });
+ });
+
+ test('transformRequestFunction can return a new plain object instead of modifying in place', async () => {
+ const enqueued: Source[] = [];
+ const requestQueue = new RequestQueue({ id: 'xxx', client: apifyClient });
+
+ // Custom mock that checks for Request instances - we override addRequestsBatched
+ // to verify that request options returned by transformRequestFunction are converted to Request instances
+ requestQueue.addRequestsBatched = async (requests) => {
+ // @ts-expect-error Iterating over the requests parameter which has a narrower type in the override
+ for (const request of requests) {
+ // This check ensures that request options are converted to Request instances
+ if (!(request instanceof Request)) {
+ throw new Error(
+ `Expected Request instance but got plain object: ${JSON.stringify(request)}`,
+ );
+ }
+ enqueued.push(request);
+ }
+ return { addedRequests: [], waitForAllRequestsToBeAdded: Promise.resolve([]) };
+ };
+
+ await cheerioCrawlerEnqueueLinks({
+ options: {
+ selector: '.click',
+ globs: ['https://example.com/**/*'],
+ transformRequestFunction: (request) => {
+ // Return a new plain object instead of modifying in place
+ return {
+ url: request.url,
+ method: 'DELETE' as const,
+ userData: { replaced: true },
+ };
+ },
+ },
+ $,
+ requestQueue,
+ originalRequestUrl: 'https://example.com',
+ });
+
+ expect(enqueued).toHaveLength(2);
+ // The request options should be properly converted to a Request
+ expect(enqueued[0].url).toBe('https://example.com/a/b/first');
+ expect(enqueued[0].method).toBe('DELETE');
+ expect(enqueued[0].userData).toEqual({ replaced: true });
+ expect(enqueued[1].url).toBe('https://example.com/a/b/third');
+ expect(enqueued[1].method).toBe('DELETE');
+ expect(enqueued[1].userData).toEqual({ replaced: true });
+ });
+
+ test('transformRequestFunction supports "skip" and "unchanged" string returns', async () => {
+ const { enqueued, requestQueue } = createRequestQueueMock();
+ const onSkippedRequest = vi.fn();
+
+ await cheerioCrawlerEnqueueLinks({
+ options: {
+ selector: '.click',
+ label: 'global-label',
+ globs: ['https://example.com/**/*'],
+ transformRequestFunction: (request) => {
+ if (request.url.includes('/a/b/first')) {
+ return 'skip';
+ }
+ // 'unchanged' should keep the original options (including global label)
+ return 'unchanged';
+ },
+ onSkippedRequest,
+ },
+ $,
+ requestQueue,
+ originalRequestUrl: 'https://example.com',
+ });
+
+ // 'skip' should exclude /a/b/first, 'unchanged' should keep /a/b/third as-is
+ expect(enqueued).toHaveLength(1);
+ expect(enqueued[0].url).toBe('https://example.com/a/b/third');
+ expect(enqueued[0].userData).toEqual({ label: 'global-label' });
+
+ // 'skip' should trigger onSkippedRequest with reason 'transform'
+ const skippedCalls = onSkippedRequest.mock.calls.map(
+ (call: unknown[]) => call[0] as { url: string; reason: string },
+ );
+ const transformSkipped = skippedCalls.filter((s) => s.url === 'https://example.com/a/b/first');
+ expect(transformSkipped).toHaveLength(1);
+ expect(transformSkipped[0]).toEqual({
+ url: 'https://example.com/a/b/first',
+ reason: 'transform',
+ });
+ // 'unchanged' should NOT trigger onSkippedRequest
+ const unchangedSkipped = skippedCalls.filter((s) => s.url === 'https://example.com/a/b/third');
+ expect(unchangedSkipped).toHaveLength(0);
+ });
+
+ test('transformRequestFunction returning falsy correctly triggers onSkippedRequest', async () => {
+ const { enqueued, requestQueue } = createRequestQueueMock();
+ const onSkippedRequest = vi.fn();
+
+ await cheerioCrawlerEnqueueLinks({
+ options: {
+ selector: '.click',
+ globs: ['https://example.com/**/*'],
+ transformRequestFunction: (request) => {
+ // Skip the first URL, keep the second
+ if (request.url.includes('/a/b/first')) {
+ return false;
+ }
+ return request;
+ },
+ onSkippedRequest,
+ },
+ $,
+ requestQueue,
+ originalRequestUrl: 'https://example.com',
+ });
+
+ expect(enqueued).toHaveLength(1);
+ expect(enqueued[0].url).toBe('https://example.com/a/b/third');
+
+ // onSkippedRequest fires for URLs filtered out by globs (another.com, cool.com)
+ // AND for the URL explicitly skipped by transformRequestFunction
+ const skippedCalls = onSkippedRequest.mock.calls.map(
+ (call: unknown[]) => call[0] as { url: string; reason: string },
+ );
+ const transformSkipped = skippedCalls.filter((s) => s.url === 'https://example.com/a/b/first');
+ expect(transformSkipped).toHaveLength(1);
+ expect(transformSkipped[0]).toEqual({
+ url: 'https://example.com/a/b/first',
+ reason: 'transform',
+ });
+ });
+ });
});
describe('respectRobotsTxtFile option', () => {
diff --git a/test/core/enqueue_links/shared.test.ts b/test/core/enqueue_links/shared.test.ts
index 8264e907d..fbb1cafd0 100644
--- a/test/core/enqueue_links/shared.test.ts
+++ b/test/core/enqueue_links/shared.test.ts
@@ -1,10 +1,11 @@
-import type { Request } from '@crawlee/core';
+import type { RequestOptions } from '@crawlee/core';
import {
+ applyRequestTransform,
constructGlobObjectsFromGlobs,
constructRegExpObjectsFromPseudoUrls,
constructRegExpObjectsFromRegExps,
createRequestOptions,
- createRequests,
+ filterRequestOptionsByPatterns,
validateGlobPattern,
} from '@crawlee/core';
@@ -72,34 +73,35 @@ describe('Enqueue links shared functions', () => {
});
});
- describe('createRequests()', () => {
- test('should work', () => {
+ describe('filterRequestOptionsByPatterns() + applyRequestTransform()', () => {
+ test('should filter by patterns and apply transform', () => {
const sources = [
'http://example.com/foo',
- { url: 'https://example.com/bar', method: 'POST', label: 'POST-REQUEST' },
+ { url: 'https://example.com/bar', method: 'POST' as const, label: 'POST-REQUEST' },
'https://apify.com',
];
const pseudoUrls = [{ purl: 'http[s?]://example.com/[.*]', userData: { one: 1 } }];
const urlPatternObjects = constructRegExpObjectsFromPseudoUrls(pseudoUrls);
- const transformRequestFunction = (request: Request) => {
- request.userData.foo = 'bar';
+ const transformRequestFunction = (request: RequestOptions) => {
+ request.userData = { ...request.userData, foo: 'bar' };
return request;
};
const requestOptions = createRequestOptions(sources);
- const requests = createRequests(requestOptions, urlPatternObjects)
- .map(transformRequestFunction)
- .filter((r) => !!r);
+ const filtered = filterRequestOptionsByPatterns(requestOptions, urlPatternObjects);
+ const transformed = applyRequestTransform(filtered, transformRequestFunction);
- expect(requests).toHaveLength(2);
- requests.forEach((r) => {
+ expect(transformed).toHaveLength(2);
+ transformed.forEach((r) => {
expect(r.url).toMatch(/^https?:\/\/example\.com\//);
expect(r.userData).toMatchObject({ foo: 'bar', one: 1 });
});
- expect(requests[0].method).toBe('GET');
- expect(requests[1].method).toBe('POST');
- expect(requests[1].userData).toEqual({ foo: 'bar', one: 1, label: 'POST-REQUEST' });
+ expect(transformed[0].method).toBeUndefined(); // defaults to GET when Request is constructed
+ expect(transformed[1].method).toBe('POST');
+ // Pattern-level userData { one: 1 } overwrites the source's userData { label: 'POST-REQUEST' },
+ // then the transform adds { foo: 'bar' }
+ expect(transformed[1].userData).toEqual({ foo: 'bar', one: 1 });
});
});