Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
fix: respect `exclude` option in `enqueueLinksByClickingElements` (#3058)
Jindřich Bär013eb028b6ec
diff --git a/packages/core/src/enqueue_links/enqueue_links.ts b/packages/core/src/enqueue_links/enqueue_links.tsindex 97c51d6ba..8c9106389 100644--- a/packages/core/src/enqueue_links/enqueue_links.ts+++ b/packages/core/src/enqueue_links/enqueue_links.ts@@ -87,7 +87,6 @@ export interface EnqueueLinksOptions extends RequestQueueOperationOptions {* containing patterns matching URLs that will **never** be enqueued.** The plain objects must include either the `glob` property or the `regexp` property.- * All remaining keys will be used as request options for the corresponding enqueued {@apilink Request} objects.** Glob matching is always case-insensitive.* If you need case-sensitive matching, provide a regexp.diff --git a/packages/playwright-crawler/src/internals/enqueue-links/click-elements.ts b/packages/playwright-crawler/src/internals/enqueue-links/click-elements.tsindex 2a12000fb..bd5acf278 100644--- a/packages/playwright-crawler/src/internals/enqueue-links/click-elements.ts+++ b/packages/playwright-crawler/src/internals/enqueue-links/click-elements.ts@@ -71,6 +71,17 @@ export interface EnqueueLinksByClickingElementsOptions {*/globs?: GlobInput[];+ /**+ * An array of glob pattern strings, regexp patterns or plain objects+ * containing patterns matching URLs that will **never** be enqueued.+ *+ * The plain objects must include either the `glob` property or the `regexp` property.+ *+ * Glob matching is always case-insensitive.+ * If you need case-sensitive matching, provide a regexp.+ */+ exclude?: readonly (GlobInput | RegExpInput)[];+/*** An array of regular expressions or plain objects* containing regular expressions matching the URLs to be enqueued.@@ -225,6 +236,9 @@ export async function enqueueLinksByClickingElements(pseudoUrls: ow.optional.array.ofType(ow.any(ow.string, ow.object.hasKeys('purl'))),globs: ow.optional.array.ofType(ow.any(ow.string, ow.object.hasKeys('glob'))),regexps: ow.optional.array.ofType(ow.any(ow.regExp, ow.object.hasKeys('regexp'))),+ exclude: ow.optional.array.ofType(+ ow.any(ow.string, ow.regExp, ow.object.hasKeys('glob'), ow.object.hasKeys('regexp')),+ ),transformRequestFunction: ow.optional.function,waitForPageIdleSecs: ow.optional.number,maxWaitForPageIdleSecs: ow.optional.number,@@ -246,13 +260,25 @@ export async function enqueueLinksByClickingElements(waitForPageIdleSecs = 1,maxWaitForPageIdleSecs = 5,forefront,+ exclude,} = options;const waitForPageIdleMillis = waitForPageIdleSecs * 1000;const maxWaitForPageIdleMillis = maxWaitForPageIdleSecs * 1000;+ const urlExcludePatternObjects: UrlPatternObject[] = [];const urlPatternObjects: UrlPatternObject[] = [];+ if (exclude?.length) {+ for (const excl of exclude) {+ if (typeof excl === 'string' || 'glob' in excl) {+ urlExcludePatternObjects.push(...constructGlobObjectsFromGlobs([excl]));+ } else if (excl instanceof RegExp || 'regexp' in excl) {+ urlExcludePatternObjects.push(...constructRegExpObjectsFromRegExps([excl]));+ }+ }+ }+if (pseudoUrls?.length) {log.deprecated('`pseudoUrls` option is deprecated, use `globs` or `regexps` instead');urlPatternObjects.push(...constructRegExpObjectsFromPseudoUrls(pseudoUrls));@@ -277,7 +303,7 @@ export async function enqueueLinksByClickingElements(if (transformRequestFunction) {requestOptions = requestOptions.map(transformRequestFunction).filter((r) => !!r) as RequestOptions[];}- const requests = createRequests(requestOptions, urlPatternObjects);+ const requests = createRequests(requestOptions, urlPatternObjects, urlExcludePatternObjects);const { addedRequests } = await requestQueue.addRequestsBatched(requests, { forefront });return { processedRequests: addedRequests, unprocessedRequests: [] };diff --git a/packages/puppeteer-crawler/src/internals/enqueue-links/click-elements.ts b/packages/puppeteer-crawler/src/internals/enqueue-links/click-elements.tsindex 73347ae90..2efefafde 100644--- a/packages/puppeteer-crawler/src/internals/enqueue-links/click-elements.ts+++ b/packages/puppeteer-crawler/src/internals/enqueue-links/click-elements.ts@@ -71,6 +71,17 @@ export interface EnqueueLinksByClickingElementsOptions {*/globs?: GlobInput[];+ /**+ * An array of glob pattern strings, regexp patterns or plain objects+ * containing patterns matching URLs that will **never** be enqueued.+ *+ * The plain objects must include either the `glob` property or the `regexp` property.+ *+ * Glob matching is always case-insensitive.+ * If you need case-sensitive matching, provide a regexp.+ */+ exclude?: readonly (GlobInput | RegExpInput)[];+/*** An array of regular expressions or plain objects* containing regular expressions matching the URLs to be enqueued.@@ -225,6 +236,9 @@ export async function enqueueLinksByClickingElements(pseudoUrls: ow.optional.array.ofType(ow.any(ow.string, ow.object.hasKeys('purl'))),globs: ow.optional.array.ofType(ow.any(ow.string, ow.object.hasKeys('glob'))),regexps: ow.optional.array.ofType(ow.any(ow.regExp, ow.object.hasKeys('regexp'))),+ exclude: ow.optional.array.ofType(+ ow.any(ow.string, ow.regExp, ow.object.hasKeys('glob'), ow.object.hasKeys('regexp')),+ ),transformRequestFunction: ow.optional.function,waitForPageIdleSecs: ow.optional.number,maxWaitForPageIdleSecs: ow.optional.number,@@ -246,13 +260,25 @@ export async function enqueueLinksByClickingElements(waitForPageIdleSecs = 1,maxWaitForPageIdleSecs = 5,forefront,+ exclude,} = options;const waitForPageIdleMillis = waitForPageIdleSecs * 1000;const maxWaitForPageIdleMillis = maxWaitForPageIdleSecs * 1000;+ const urlExcludePatternObjects: UrlPatternObject[] = [];const urlPatternObjects: UrlPatternObject[] = [];+ if (exclude?.length) {+ for (const excl of exclude) {+ if (typeof excl === 'string' || 'glob' in excl) {+ urlExcludePatternObjects.push(...constructGlobObjectsFromGlobs([excl]));+ } else if (excl instanceof RegExp || 'regexp' in excl) {+ urlExcludePatternObjects.push(...constructRegExpObjectsFromRegExps([excl]));+ }+ }+ }+if (pseudoUrls?.length) {log.deprecated('`pseudoUrls` option is deprecated, use `globs` or `regexps` instead');urlPatternObjects.push(...constructRegExpObjectsFromPseudoUrls(pseudoUrls));@@ -277,7 +303,7 @@ export async function enqueueLinksByClickingElements(if (transformRequestFunction) {requestOptions = requestOptions.map(transformRequestFunction).filter((r) => !!r) as RequestOptions[];}- const requests = createRequests(requestOptions, urlPatternObjects);+ const requests = createRequests(requestOptions, urlPatternObjects, urlExcludePatternObjects);const { addedRequests } = await requestQueue.addRequestsBatched(requests, { forefront });return { processedRequests: addedRequests, unprocessedRequests: [] };