Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
feat(core): pass Request to onSkippedRequest (#3992)
Jindřich Bärd26a7f64abb5
diff --git a/docs/public-api/crawlee-core.api.md b/docs/public-api/crawlee-core.api.mdindex 785777d49..f324e51f8 100644--- a/docs/public-api/crawlee-core.api.md+++ b/docs/public-api/crawlee-core.api.md@@ -1797,7 +1797,7 @@ export interface SitemapRequestLoaderOptions extends UrlConstraints {// @public (undocumented)export type SkippedRequestCallback = (args: {- url: string;+ request: Request_2;reason: SkippedRequestReason;}) => Awaitable<void>;diff --git a/docs/upgrading/upgrading_v4.md b/docs/upgrading/upgrading_v4.mdindex b7312d3e5..1dba7cc36 100644--- a/docs/upgrading/upgrading_v4.md+++ b/docs/upgrading/upgrading_v4.md@@ -843,6 +843,10 @@ const urls = await extractLinks({ selector: '.product-link' });The `robotsTxtFile` / `respectRobotsTxtFile` per-call options are removed from `enqueueLinks()` — robots.txt filtering is applied by the crawler consistently via `BasicCrawlerOptions.respectRobotsTxtFile`.+### `onSkippedRequest` receives a `Request` instead of a URL string++The callback now gets `{ request, reason }` instead of `{ url, reason }` — use `request.url` for the URL.+### Internal KVS keys renamedSeveral internal Crawlee keys were prefixed with the `SDK_` prefix for legacy reasons — these keys now start with `CRAWLEE_` instead. These are, e.g., `CRAWLEE_SESSION_POOL_STATE` or `CRAWLEE_CRAWLER_STATISTICS_{n}`.diff --git a/packages/basic-crawler/src/internals/basic-crawler.ts b/packages/basic-crawler/src/internals/basic-crawler.tsindex 64c7923ab..a3d839947 100644--- a/packages/basic-crawler/src/internals/basic-crawler.ts+++ b/packages/basic-crawler/src/internals/basic-crawler.ts@@ -18,6 +18,7 @@ import type {IRequestLoader,IRequestManager,IStatistics,+ RequestOptions,RequestsLike,RouterHandler,RouterRoutes,@@ -45,6 +46,7 @@ import {ContextPipelineInitializationError,ContextPipelineInterruptedError,createRequestOptions,+ createSkippedRequestArgs,createStorageTransaction,Request,CriticalError,@@ -1454,7 +1456,7 @@ export class BasicCrawler<request.state = RequestState.SKIPPED;request.noRetry = true;await this.#handleSkippedRequest({- url: request.url,+ request,reason: 'robotsTxt',});@@ -1587,7 +1589,7 @@ export class BasicCrawler<request.noRetry = true;request.state = RequestState.SKIPPED;- await this.#handleSkippedRequest({ url: request.url, reason: 'redirect' });+ await this.#handleSkippedRequest({ request, reason: 'redirect' });throw new ContextPipelineInterruptedError(message);}@@ -2099,7 +2101,12 @@ export class BasicCrawler<const maxCrawlDepth = this.#maxCrawlDepth;const validateRequestUserData = this.validateRequestUserData.bind(this);- const allSkipped: { url: string; reason: SkippedRequestReason }[] = [];+ const allSkipped: { source: string | Source; reason: SkippedRequestReason }[] = [];+ // A skipped source (which can carry arbitrary userData) is only retained if something reads it -+ // otherwise the URL alone is enough to build the callback argument and to log with.+ const hasSkippedRequestCallback =+ this.#onSkippedRequest !== undefined || options.onSkippedRequest !== undefined;+ const keepSkippedSource = (source: Source) => (hasSkippedRequestCallback ? source : source.url!);async function* filteredRequests() {for await (const request of requests) {@@ -2113,16 +2120,17 @@ export class BasicCrawler<}if (maxCrawlDepth !== undefined && requestOptions.crawlDepth! > maxCrawlDepth) {- allSkipped.push({ url: requestOptions.url, reason: 'depth' });+ allSkipped.push({ source: keepSkippedSource(requestOptions), reason: 'depth' });continue;}if (!(await isAllowedBasedOnRobotsTxtFile(requestOptions.url))) {- allSkipped.push({ url: requestOptions.url, reason: 'robotsTxt' });+ allSkipped.push({ source: keepSkippedSource(requestOptions), reason: 'robotsTxt' });continue;}- const onSkippedFilterUrl = (url: string) => allSkipped.push({ url, reason: 'filters' });+ const onSkippedByFilter = (opts: RequestOptions) =>+ allSkipped.push({ source: keepSkippedSource(opts), reason: 'filters' });// Filter by user patterns first (with exclude)...let filtered = filterRequestOptionsByPatterns(@@ -2130,7 +2138,7 @@ export class BasicCrawler<urlPatternObjects.length > 0 ? urlPatternObjects : undefined,urlExcludePatternObjects,strategy,- onSkippedFilterUrl,+ onSkippedByFilter,);// ...then filter by the enqueue strategy (making this an AND check)filtered = filterRequestOptionsByPatterns(@@ -2138,7 +2146,7 @@ export class BasicCrawler<enqueueStrategyPatterns.length > 0 ? enqueueStrategyPatterns : undefined,[],strategy,- onSkippedFilterUrl,+ onSkippedByFilter,);if (filtered.length === 0) {@@ -2149,7 +2157,7 @@ export class BasicCrawler<if (options.transformRequestFunction) {const transformed = applyRequestTransform([finalOptions], options.transformRequestFunction, (r) =>- allSkipped.push({ url: r.url, reason: r.skippedReason ?? 'transform' }),+ allSkipped.push({ source: keepSkippedSource(r), reason: r.skippedReason ?? 'transform' }),);if (transformed.length === 0) {@@ -2175,11 +2183,13 @@ export class BasicCrawler<// Report requests skipped due to the maxNewRequests budget (i.e. maxRequestsPerCrawl limit, or an// explicit `limit` option)for (const request of result.requestsOverLimit ?? []) {- allSkipped.push({ url: typeof request === 'string' ? request : request.url!, reason: 'limit' });+ allSkipped.push({ source: request, reason: 'limit' });}if (allSkipped.length > 0) {- const skippedRobotsUrls = allSkipped.filter((s) => s.reason === 'robotsTxt').map((s) => s.url);+ const skippedRobotsUrls = allSkipped+ .filter((s) => s.reason === 'robotsTxt')+ .map(({ source }) => (typeof source === 'string' ? source : source.url!));if (skippedRobotsUrls.length > 0) {this.log.warning(`Some requests were skipped because they were disallowed based on the robots.txt file`,@@ -2198,9 +2208,10 @@ export class BasicCrawler<}await Promise.all(- allSkipped.map(async ({ url, reason }) => {- await this.#handleSkippedRequest({ url, reason });- await options.onSkippedRequest?.({ url, reason });+ allSkipped.map(async ({ source, reason }) => {+ const args = createSkippedRequestArgs(source, reason);+ await this.#handleSkippedRequest(args);+ await options.onSkippedRequest?.(args);}),);}diff --git a/packages/core/src/enqueue_links/shared.ts b/packages/core/src/enqueue_links/shared.tsindex f86fb9d47..b1804ecf4 100644--- a/packages/core/src/enqueue_links/shared.ts+++ b/packages/core/src/enqueue_links/shared.ts@@ -4,7 +4,8 @@ import type { Awaitable, Dictionary } from '@crawlee/types';import { Minimatch } from 'minimatch';import { z } from 'zod';-import type { RequestOptions } from '../request.js';+import type { RequestOptions, Source } from '../request.js';+import { Request } from '../request.js';import { schemas } from '../validators.js';import type { EnqueueStrategyOption } from './enqueue_links.js';@@ -57,7 +58,32 @@ export type SkippedRequestReason =| 'redirect'| 'depth';-export type SkippedRequestCallback = (args: { url: string; reason: SkippedRequestReason }) => Awaitable<void>;+export type SkippedRequestCallback = (args: { request: Request; reason: SkippedRequestReason }) => Awaitable<void>;++/**+ * Builds the `{ request, reason }` argument passed to a {@apilink SkippedRequestCallback}, constructing the+ * `Request` lazily on first access to `request` (and caching it) since most skips are never observed by a+ * real callback and building a `Request` isn't free.+ * @ignore+ */+export function createSkippedRequestArgs(+ source: string | Source,+ reason: SkippedRequestReason,+): Parameters<SkippedRequestCallback>[0] {+ const sourceReason = typeof source === 'string' ? undefined : source.skippedReason;+ let request: Request | undefined;++ return {+ reason: sourceReason ?? reason,+ get request() {+ request ??=+ source instanceof Request+ ? source+ : new Request(typeof source === 'string' ? { url: source } : (source as RequestOptions));+ return request;+ },+ };+}/*** @ignore@@ -176,16 +202,16 @@ export function filterRequestOptionsByPatterns(includePatterns: UrlPatternObject[] | undefined,excludePatterns: UrlPatternObject[] = [],strategy?: EnqueueStrategyOption,- onSkippedUrl?: (url: string) => void,+ onSkippedRequestOptions?: (options: RequestOptions) => void,): RequestOptions[] {const excludeMatchers = excludePatterns.map(createPatternObjectMatcher);const includeMatchers = includePatterns?.length ? includePatterns.map(createPatternObjectMatcher) : undefined;return requestOptions- .filter(({ url }) => {- const matchesExclude = excludeMatchers.some(({ match }) => match(url));+ .filter((opts) => {+ const matchesExclude = excludeMatchers.some(({ match }) => match(opts.url));if (matchesExclude) {- onSkippedUrl?.(url);+ onSkippedRequestOptions?.(opts);}return !matchesExclude;})@@ -201,7 +227,7 @@ export function filterRequestOptionsByPatterns(}// didn't match any positive pattern- onSkippedUrl?.(opts.url);+ onSkippedRequestOptions?.(opts);return null;}).filter((opts) => opts !== null);diff --git a/packages/playwright-crawler/src/internals/enqueue-links/click-elements.ts b/packages/playwright-crawler/src/internals/enqueue-links/click-elements.tsindex 155043fbd..e5cbcd969 100644--- a/packages/playwright-crawler/src/internals/enqueue-links/click-elements.ts+++ b/packages/playwright-crawler/src/internals/enqueue-links/click-elements.ts@@ -12,6 +12,7 @@ import {applyRequestTransform,constructUrlPatternObjects,createRequestOptions,+ createSkippedRequestArgs,filterRequestOptionsByPatterns,urlPatternSchema,Request as CrawleeRequest,@@ -236,6 +237,7 @@ export async function enqueueLinksByClickingElements(const waitForPageIdleMillis = waitForPageIdleSecs * 1000;const maxWaitForPageIdleMillis = maxWaitForPageIdleSecs * 1000;+ const hasOnSkippedRequest = onSkippedRequest !== undefined;const urlExcludePatternObjects: UrlPatternObject[] = exclude?.length ? constructUrlPatternObjects(exclude) : [];const urlPatternObjects: UrlPatternObject[] = include?.length ? constructUrlPatternObjects(include) : [];@@ -248,27 +250,31 @@ export async function enqueueLinksByClickingElements(clickOptions,});const requestOptions = createRequestOptions(interceptedRequests, parsedOptions);- const skippedByFilters: string[] = [];+ const skippedByFilters: RequestOptions[] = [];let filteredOptions = filterRequestOptionsByPatterns(requestOptions,urlPatternObjects.length > 0 ? urlPatternObjects : undefined,urlExcludePatternObjects,undefined,- (url) => skippedByFilters.push(url),+ hasOnSkippedRequest ? (opts) => skippedByFilters.push(opts) : undefined,);if (onSkippedRequest && skippedByFilters.length > 0) {- await Promise.all(skippedByFilters.map(async (url) => onSkippedRequest({ url, reason: 'filters' })));+ await Promise.all(+ skippedByFilters.map(async (opts) => onSkippedRequest(createSkippedRequestArgs(opts, 'filters'))),+ );}if (transformRequestFunction) {const skippedByTransform: RequestOptions[] = [];- filteredOptions = applyRequestTransform(filteredOptions, transformRequestFunction, (r) =>- skippedByTransform.push(r),+ filteredOptions = applyRequestTransform(+ filteredOptions,+ transformRequestFunction,+ hasOnSkippedRequest ? (r) => skippedByTransform.push(r) : undefined,);if (onSkippedRequest && skippedByTransform.length > 0) {await Promise.all(- skippedByTransform.map(async (r) => onSkippedRequest({ url: r.url, reason: 'transform' })),+ skippedByTransform.map(async (r) => onSkippedRequest(createSkippedRequestArgs(r, 'transform'))),);}}diff --git a/packages/puppeteer-crawler/src/internals/enqueue-links/click-elements.ts b/packages/puppeteer-crawler/src/internals/enqueue-links/click-elements.tsindex 5126c8d23..f7372962b 100644--- a/packages/puppeteer-crawler/src/internals/enqueue-links/click-elements.ts+++ b/packages/puppeteer-crawler/src/internals/enqueue-links/click-elements.ts@@ -12,6 +12,7 @@ import {applyRequestTransform,constructUrlPatternObjects,createRequestOptions,+ createSkippedRequestArgs,filterRequestOptionsByPatterns,urlPatternSchema,Request,@@ -237,6 +238,7 @@ export async function enqueueLinksByClickingElements(const waitForPageIdleMillis = waitForPageIdleSecs * 1000;const maxWaitForPageIdleMillis = maxWaitForPageIdleSecs * 1000;+ const hasOnSkippedRequest = onSkippedRequest !== undefined;const urlExcludePatternObjects: UrlPatternObject[] = exclude?.length ? constructUrlPatternObjects(exclude) : [];const urlPatternObjects: UrlPatternObject[] = include?.length ? constructUrlPatternObjects(include) : [];@@ -249,27 +251,31 @@ export async function enqueueLinksByClickingElements(clickOptions,});const requestOptions = createRequestOptions(interceptedRequests, parsedOptions);- const skippedByFilters: string[] = [];+ const skippedByFilters: RequestOptions[] = [];let filteredOptions = filterRequestOptionsByPatterns(requestOptions,urlPatternObjects.length > 0 ? urlPatternObjects : undefined,urlExcludePatternObjects,undefined,- (url) => skippedByFilters.push(url),+ hasOnSkippedRequest ? (opts) => skippedByFilters.push(opts) : undefined,);if (onSkippedRequest && skippedByFilters.length > 0) {- await Promise.all(skippedByFilters.map(async (url) => onSkippedRequest({ url, reason: 'filters' })));+ await Promise.all(+ skippedByFilters.map(async (opts) => onSkippedRequest(createSkippedRequestArgs(opts, 'filters'))),+ );}if (transformRequestFunction) {const skippedByTransform: RequestOptions[] = [];- filteredOptions = applyRequestTransform(filteredOptions, transformRequestFunction, (r) =>- skippedByTransform.push(r),+ filteredOptions = applyRequestTransform(+ filteredOptions,+ transformRequestFunction,+ hasOnSkippedRequest ? (r) => skippedByTransform.push(r) : undefined,);if (onSkippedRequest && skippedByTransform.length > 0) {await Promise.all(- skippedByTransform.map(async (r) => onSkippedRequest({ url: r.url, reason: 'transform' })),+ skippedByTransform.map(async (r) => onSkippedRequest(createSkippedRequestArgs(r, 'transform'))),);}}diff --git a/test/core/crawlers/basic_crawler.test.ts b/test/core/crawlers/basic_crawler.test.tsindex 846e4d5bb..b85f01afa 100644--- a/test/core/crawlers/basic_crawler.test.ts+++ b/test/core/crawlers/basic_crawler.test.ts@@ -535,8 +535,12 @@ describe('BasicCrawler', () => {const skippedRequests = onSkippedRequestMock.mock.calls.map((call) => call[0]);expect(skippedRequests).toHaveLength(2);- expect(skippedRequests[0]).toStrictEqual({ url: 'https://example.com/1/', reason: 'depth' });- expect(skippedRequests[1]).toStrictEqual({ url: 'https://example.com/2/', reason: 'depth' });+ expect(skippedRequests[0].reason).toBe('depth');+ expect(skippedRequests[0].request).toBeInstanceOf(Request);+ expect(skippedRequests[0].request.url).toBe('https://example.com/1/');+ expect(skippedRequests[1].reason).toBe('depth');+ expect(skippedRequests[1].request).toBeInstanceOf(Request);+ expect(skippedRequests[1].request.url).toBe('https://example.com/2/');});it('should respect user provided transformRequestFunction', async () => {@@ -565,8 +569,10 @@ describe('BasicCrawler', () => {const skippedRequests = onSkippedRequestMock.mock.calls.map((call) => call[0]);expect(skippedRequests).toHaveLength(2);- expect(skippedRequests[0]).toStrictEqual({ url: 'https://example.com/1/', reason: 'transform' });- expect(skippedRequests[1]).toStrictEqual({ url: 'https://example.com/2/', reason: 'transform' });+ expect(skippedRequests[0].reason).toBe('transform');+ expect(skippedRequests[0].request.url).toBe('https://example.com/1/');+ expect(skippedRequests[1].reason).toBe('transform');+ expect(skippedRequests[1].request.url).toBe('https://example.com/2/');},);@@ -585,8 +591,12 @@ describe('BasicCrawler', () => {// The skipped reason should be 'depth', not 'transform'const skippedRequests = onSkippedRequestMock.mock.calls.map((call) => call[0]);expect(skippedRequests).toHaveLength(2);- expect(skippedRequests[0]).toStrictEqual({ url: 'https://example.com/1/', reason: 'depth' });- expect(skippedRequests[1]).toStrictEqual({ url: 'https://example.com/2/', reason: 'depth' });+ expect(skippedRequests[0].reason).toBe('depth');+ expect(skippedRequests[0].request).toBeInstanceOf(Request);+ expect(skippedRequests[0].request.url).toBe('https://example.com/1/');+ expect(skippedRequests[1].reason).toBe('depth');+ expect(skippedRequests[1].request).toBeInstanceOf(Request);+ expect(skippedRequests[1].request.url).toBe('https://example.com/2/');});});@@ -3311,7 +3321,7 @@ describe('BasicCrawler', () => {const skippedUrls = onSkippedRequest.mock.calls.map((call) => call[0]).filter(({ reason }) => reason === 'limit')- .map(({ url }) => url)+ .map(({ request }) => request.url).sort();expect(skippedUrls).toEqual([@@ -3379,15 +3389,15 @@ describe('BasicCrawler', () => {await crawler.run(['http://example.com']);- const skipped = [- { url: 'http://example.com/2', reason: 'limit' },- { url: 'http://example.com/3', reason: 'limit' },- ];-for (const mock of [crawlerOnSkippedRequest, userOnSkippedRequest]) {- expect(mock.mock.calls.map((call) => call[0]).sort((a, b) => a.url.localeCompare(b.url))).toEqual(- skipped,- );+ const skipped = mock.mock.calls+ .map((call) => ({ url: call[0].request.url, reason: call[0].reason }))+ .sort((a, b) => a.url.localeCompare(b.url));++ expect(skipped).toEqual([+ { url: 'http://example.com/2', reason: 'limit' },+ { url: 'http://example.com/3', reason: 'limit' },+ ]);}});});@@ -3559,8 +3569,8 @@ describe('BasicCrawler', () => {maxRequestRetries: 0,respectRobotsTxtFile: true,requestHandler: async () => {},- onSkippedRequest: async ({ url, reason }) => {- await (await KeyValueStore.open()).setValue('skipped', { url, reason });+ onSkippedRequest: async ({ request, reason }) => {+ await (await KeyValueStore.open()).setValue('skipped', { url: request.url, reason });},});diff --git a/test/core/enqueue_links/enqueue_links.test.ts b/test/core/enqueue_links/enqueue_links.test.tsindex 448bc6210..06772447d 100644--- a/test/core/enqueue_links/enqueue_links.test.ts+++ b/test/core/enqueue_links/enqueue_links.test.ts@@ -667,12 +667,12 @@ describe('enqueueLinks()', () => {expect(enqueued[0].userData).toEqual({ label: 'global-label' });const skippedCalls = onSkippedRequest.mock.calls.map(- (call: unknown[]) => call[0] as { url: string; reason: string },+ (call: unknown[]) => call[0] as { request: Request; reason: string },);- const transformSkipped = skippedCalls.filter((s) => s.url === 'https://example.com/a/b/first');+ const transformSkipped = skippedCalls.filter((s) => s.request.url === 'https://example.com/a/b/first');expect(transformSkipped).toHaveLength(1);- expect(transformSkipped[0]).toEqual({ url: 'https://example.com/a/b/first', reason: 'transform' });- const unchangedSkipped = skippedCalls.filter((s) => s.url === 'https://example.com/a/b/third');+ expect(transformSkipped[0].reason).toBe('transform');+ const unchangedSkipped = skippedCalls.filter((s) => s.request.url === 'https://example.com/a/b/third');expect(unchangedSkipped).toHaveLength(0);});@@ -699,11 +699,11 @@ describe('enqueueLinks()', () => {expect(enqueued[0].url).toBe('https://example.com/a/b/third');const skippedCalls = onSkippedRequest.mock.calls.map(- (call: unknown[]) => call[0] as { url: string; reason: string },+ (call: unknown[]) => call[0] as { request: Request; reason: string },);- const transformSkipped = skippedCalls.filter((s) => s.url === 'https://example.com/a/b/first');+ const transformSkipped = skippedCalls.filter((s) => s.request.url === 'https://example.com/a/b/first');expect(transformSkipped).toHaveLength(1);- expect(transformSkipped[0]).toEqual({ url: 'https://example.com/a/b/first', reason: 'transform' });+ expect(transformSkipped[0].reason).toBe('transform');});});});diff --git a/test/e2e/adaptive-playwright-robots-file/actor/main.js b/test/e2e/adaptive-playwright-robots-file/actor/main.jsindex f2138de38..202d08d70 100644--- a/test/e2e/adaptive-playwright-robots-file/actor/main.js+++ b/test/e2e/adaptive-playwright-robots-file/actor/main.js@@ -53,7 +53,8 @@ await Actor.init({const crawler = new AdaptivePlaywrightCrawler({maxRequestsPerCrawl: 10,respectRobotsTxtFile: true,- onSkippedRequest: (args) => crawler.log.warningOnce(`Request ${args.url} was skipped, reason: ${args.reason}`),+ onSkippedRequest: (args) =>+ crawler.log.warningOnce(`Request ${args.request.url} was skipped, reason: ${args.reason}`),});crawler.router.addDefaultHandler(async ({ log, request, enqueueLinks, pushData }) => {