Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
feat: Report links skipped because of various filter conditions (#3026)
Jan Buchar5a867bc28135
diff --git a/packages/basic-crawler/src/internals/basic-crawler.ts b/packages/basic-crawler/src/internals/basic-crawler.tsindex b531ef0a6..2b0c5c431 100644--- a/packages/basic-crawler/src/internals/basic-crawler.ts+++ b/packages/basic-crawler/src/internals/basic-crawler.ts@@ -354,7 +354,11 @@ export interface BasicCrawlerOptions<Context extends CrawlingContext = BasicCraw/*** When a request is skipped for some reason, you can use this callback to act on it.- * This is currently fired only for requests skipped based on robots.txt file.+ * This is currently fired for requests skipped+ * 1. based on robots.txt file,+ * 2. because they don't match enqueueLinks filters,+ * 3. because they are redirected to a URL that doesn't match the enqueueLinks strategy,+ * 4. or because the {@apilink BasicCrawlerOptions.maxRequestsPerCrawl|`maxRequestsPerCrawl`} limit has been reached*/onSkippedRequest?: SkippedRequestCallback;diff --git a/packages/browser-crawler/src/internals/browser-crawler.ts b/packages/browser-crawler/src/internals/browser-crawler.tsindex 5424a54b6..68618f5e3 100644--- a/packages/browser-crawler/src/internals/browser-crawler.ts+++ b/packages/browser-crawler/src/internals/browser-crawler.ts@@ -573,6 +573,8 @@ export abstract class BrowserCrawler<request.noRetry = true;request.state = RequestState.SKIPPED;+ await this.handleSkippedRequest({ url: request.url, reason: 'redirect' });+return;}diff --git a/packages/core/src/enqueue_links/enqueue_links.ts b/packages/core/src/enqueue_links/enqueue_links.tsindex 4de08a56a..e0912d83f 100644--- a/packages/core/src/enqueue_links/enqueue_links.ts+++ b/packages/core/src/enqueue_links/enqueue_links.ts@@ -1,4 +1,4 @@-import type { Awaitable, BatchAddRequestsResult, Dictionary } from '@crawlee/types';+import type { BatchAddRequestsResult, Dictionary } from '@crawlee/types';import { type RobotsTxtFile } from '@crawlee/utils';import ow from 'ow';import { getDomain } from 'tldts';@@ -13,7 +13,15 @@ import type {RequestProvider,RequestQueueOperationOptions,} from '../storages';-import type { GlobInput, PseudoUrlInput, RegExpInput, RequestTransform, UrlPatternObject } from './shared';+import type {+ GlobInput,+ PseudoUrlInput,+ RegExpInput,+ RequestTransform,+ SkippedRequestCallback,+ SkippedRequestReason,+ UrlPatternObject,+} from './shared';import {constructGlobObjectsFromGlobs,constructRegExpObjectsFromPseudoUrls,@@ -23,8 +31,6 @@ import {filterRequestsByPatterns,} from './shared';-export type SkippedRequestCallback = (args: { url: string; reason: 'robotsTxt' | 'limit' }) => Awaitable<void>;-export interface EnqueueLinksOptions extends RequestQueueOperationOptions {/** Limit the amount of actually enqueued URLs to this number. Useful for testing across the entire crawling scope. */limit?: number;@@ -175,7 +181,10 @@ export interface EnqueueLinksOptions extends RequestQueueOperationOptions {/*** When a request is skipped for some reason, you can use this callback to act on it.- * This is currently fired only for requests skipped based on robots.txt file.+ * This is currently fired for requests skipped+ * 1. based on robots.txt file,+ * 2. because they don't match enqueueLinks filters,+ * 3. or because the maxRequestsPerCrawl limit has been reached*/onSkippedRequest?: SkippedRequestCallback;}@@ -392,6 +401,16 @@ export async function enqueueLinks(}}+ async function reportSkippedRequests(skippedRequests: { url: string }[], reason: SkippedRequestReason) {+ if (onSkippedRequest && skippedRequests.length > 0) {+ await Promise.all(+ skippedRequests.map((request) => {+ return onSkippedRequest({ url: request.url, reason });+ }),+ );+ }+ }+let requestOptions = createRequestOptions(urls, options);if (robotsTxtFile) {@@ -406,25 +425,37 @@ export async function enqueueLinks(return false;});- if (onSkippedRequest && skippedRequests.length > 0) {- await Promise.all(- skippedRequests.map((request) => {- return onSkippedRequest({ url: request.url, reason: 'robotsTxt' });- }),- );- }+ await reportSkippedRequests(skippedRequests, 'robotsTxt');}if (transformRequestFunction) {+ const skippedRequests: RequestOptions[] = [];+requestOptions = requestOptions- .map((request) => transformRequestFunction(request))- .filter((r) => !!r) as RequestOptions[];+ .map((request) => {+ const transformedRequest = transformRequestFunction(request);+ if (!transformedRequest) {+ skippedRequests.push(request);+ }+ return transformedRequest;+ })+ .filter((r) => Boolean(r)) as RequestOptions[];++ await reportSkippedRequests(skippedRequests, 'filters');}- function createFilteredRequests() {+ async function createFilteredRequests() {+ const skippedRequests: string[] = [];+// No user provided patterns means we can skip an extra filtering stepif (urlPatternObjects.length === 0) {- return createRequests(requestOptions, enqueueStrategyPatterns, urlExcludePatternObjects, options.strategy);+ return createRequests(+ requestOptions,+ enqueueStrategyPatterns,+ urlExcludePatternObjects,+ options.strategy,+ (url) => skippedRequests.push(url),+ );}// Generate requests based on the user patterns first@@ -433,19 +464,24 @@ export async function enqueueLinks(urlPatternObjects,urlExcludePatternObjects,options.strategy,+ (url) => skippedRequests.push(url),);// ...then filter them by the enqueue links strategy (making this an AND check)- return filterRequestsByPatterns(generatedRequestsFromUserFilters, enqueueStrategyPatterns);+ const filtered = filterRequestsByPatterns(generatedRequestsFromUserFilters, enqueueStrategyPatterns, (url) =>+ skippedRequests.push(url),+ );++ await reportSkippedRequests(+ skippedRequests.map((url) => ({ url })),+ 'filters',+ );++ return filtered;}- let requests = createFilteredRequests();+ let requests = await createFilteredRequests();if (limit && limit < requests.length) {- if (onSkippedRequest) {- for (const request of requests.slice(limit)) {- await onSkippedRequest({ url: request.url, reason: 'limit' });- }- }-+ await reportSkippedRequests(requests.slice(limit), 'limit');requests = requests.slice(0, limit);}diff --git a/packages/core/src/enqueue_links/shared.ts b/packages/core/src/enqueue_links/shared.tsindex 72c04a676..ffc3b643b 100644--- a/packages/core/src/enqueue_links/shared.ts+++ b/packages/core/src/enqueue_links/shared.ts@@ -1,5 +1,6 @@import { URL } from 'node:url';+import type { Awaitable } from '@crawlee/types';import { minimatch } from 'minimatch';import { purlToRegExp } from '@apify/pseudo_url';@@ -46,6 +47,13 @@ export type RegExpObject = { regexp: RegExp } & Pick<export type RegExpInput = RegExp | RegExpObject;+export type SkippedRequestReason = 'robotsTxt' | 'limit' | 'filters' | 'redirect';++export type SkippedRequestCallback = (args: {+ url: string;+ reason: SkippedRequestReason;+}) => Awaitable<void>;+/*** @ignore*/@@ -166,14 +174,21 @@ export function createRequests(urlPatternObjects?: UrlPatternObject[],excludePatternObjects: UrlPatternObject[] = [],strategy?: EnqueueLinksOptions['strategy'],+ onSkippedUrl?: (url: string) => void,): Request[] {return requestOptions.map((opts) => ({ url: typeof opts === 'string' ? opts : opts.url, opts })).filter(({ url }) => {- return !excludePatternObjects.some((excludePatternObject) => {+ const matchesExcludePatterns = excludePatternObjects.some((excludePatternObject) => {const { regexp, glob } = excludePatternObject;return (regexp && url.match(regexp)) || (glob && minimatch(url, glob, { nocase: true }));});++ if (matchesExcludePatterns) {+ onSkippedUrl?.(url);+ }++ return !matchesExcludePatterns;}).map(({ url, opts }) => {if (!urlPatternObjects || !urlPatternObjects.length) {@@ -193,12 +208,17 @@ export function createRequests(}// didn't match any positive pattern+ onSkippedUrl?.(url);return null;}).filter((request) => request) as Request[];}-export function filterRequestsByPatterns(requests: Request[], patterns?: UrlPatternObject[]): Request[] {+export function filterRequestsByPatterns(+ requests: Request[],+ patterns?: UrlPatternObject[],+ onSkippedUrl?: (url: string) => void,+): Request[] {if (!patterns?.length) {return requests;}@@ -206,14 +226,15 @@ export function filterRequestsByPatterns(requests: Request[], patterns?: UrlPattconst filtered: Request[] = [];for (const request of requests) {- for (const urlPatternObject of patterns) {- const { regexp, glob } = urlPatternObject;+ const matchingPattern = patterns.find(+ ({ regexp, glob }) =>+ (regexp && request.url.match(regexp)) || (glob && minimatch(request.url, glob, { nocase: true })),+ );- if ((regexp && request.url.match(regexp)) || (glob && minimatch(request.url, glob, { nocase: true }))) {- filtered.push(request);- // Break the pattern loop, as we already matched this request once- break;- }+ if (matchingPattern !== undefined) {+ filtered.push(request);+ } else {+ onSkippedUrl?.(request.url);}}diff --git a/packages/http-crawler/src/internals/http-crawler.ts b/packages/http-crawler/src/internals/http-crawler.tsindex 8dea50e53..1811a6b35 100644--- a/packages/http-crawler/src/internals/http-crawler.ts+++ b/packages/http-crawler/src/internals/http-crawler.ts@@ -557,6 +557,8 @@ export class HttpCrawler<request.noRetry = true;request.state = RequestState.SKIPPED;+ await this.handleSkippedRequest({ url: request.url, reason: 'redirect' });+return;}diff --git a/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.ts b/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.tsindex f01897285..6208fa5c6 100644--- a/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.ts+++ b/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.ts@@ -600,6 +600,8 @@ export class AdaptivePlaywrightCrawler extends PlaywrightCrawler {request.noRetry = true;request.state = RequestState.SKIPPED;+ await this.handleSkippedRequest({ url: request.url, reason: 'redirect' });+return;}