Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

feat(core): add `exclude` option to `enqueueLinks` (#1786)

Jack Hedaya2e833dc4b0b8
diff --git a/packages/core/src/enqueue_links/enqueue_links.ts b/packages/core/src/enqueue_links/enqueue_links.ts
index 3a2d6ab1d..4504f70c0 100644
--- a/packages/core/src/enqueue_links/enqueue_links.ts
+++ b/packages/core/src/enqueue_links/enqueue_links.ts
@@ -55,6 +55,18 @@ export interface EnqueueLinksOptions extends RequestQueueOperationOptions {
*/
globs?: GlobInput[];
+ /**
+ * An array of glob pattern strings, regexp patterns or plain objects
+ * containing patterns matching URLs that will **never** be enqueued.
+ *
+ * The plain objects must include either the `glob` property or the `regexp` property.
+ * All remaining keys will be used as request options for the corresponding enqueued {@apilink Request} objects.
+ *
+ * Glob matching is always case-insensitive.
+ * If you need case-sensitive matching, provide a regexp.
+ */
+ exclude?: (GlobInput | RegExpInput)[];
+
/**
* An array of regular expressions or plain objects
* containing regular expressions matching the URLs to be enqueued.
@@ -239,6 +251,12 @@ export async function enqueueLinks(options: SetRequired<EnqueueLinksOptions, 're
ow.string,
ow.object.hasKeys('glob'),
)),
+ exclude: ow.optional.array.ofType(ow.any(
+ ow.string,
+ ow.regExp,
+ ow.object.hasKeys('glob'),
+ ow.object.hasKeys('regexp'),
+ )),
regexps: ow.optional.array.ofType(ow.any(
ow.regExp,
ow.object.hasKeys('regexp'),
@@ -252,14 +270,26 @@ export async function enqueueLinks(options: SetRequired<EnqueueLinksOptions, 're
limit,
urls,
pseudoUrls,
+ exclude,
globs,
regexps,
transformRequestFunction,
forefront,
} = options;
+ const urlExcludePatternObjects: UrlPatternObject[] = [];
const urlPatternObjects: UrlPatternObject[] = [];
+ if (exclude?.length) {
+ for (const excl of exclude) {
+ if (typeof excl === 'string' || 'glob' in excl) {
+ urlExcludePatternObjects.push(...constructGlobObjectsFromGlobs([excl]));
+ } else if (excl instanceof RegExp || 'regexp' in excl) {
+ urlExcludePatternObjects.push(...constructRegExpObjectsFromRegExps([excl]));
+ }
+ }
+ }
+
if (pseudoUrls?.length) {
log.deprecated('`pseudoUrls` option is deprecated, use `globs` or `regexps` instead');
urlPatternObjects.push(...constructRegExpObjectsFromPseudoUrls(pseudoUrls));
@@ -328,11 +358,11 @@ export async function enqueueLinks(options: SetRequired<EnqueueLinksOptions, 're
function createFilteredRequests() {
// No user provided patterns means we can skip an extra filtering step
if (urlPatternObjects.length === 0) {
- return createRequests(requestOptions, enqueueStrategyPatterns);
+ return createRequests(requestOptions, enqueueStrategyPatterns, urlExcludePatternObjects);
}
// Generate requests based on the user patterns first
- const generatedRequestsFromUserFilters = createRequests(requestOptions, urlPatternObjects);
+ const generatedRequestsFromUserFilters = createRequests(requestOptions, urlPatternObjects, urlExcludePatternObjects);
// ...then filter them by the enqueue links strategy (making this an AND check)
return filterRequestsByPatterns(generatedRequestsFromUserFilters, enqueueStrategyPatterns);
}
diff --git a/packages/core/src/enqueue_links/shared.ts b/packages/core/src/enqueue_links/shared.ts
index 50ebd601c..4969f2d22 100644
--- a/packages/core/src/enqueue_links/shared.ts
+++ b/packages/core/src/enqueue_links/shared.ts
@@ -126,7 +126,11 @@ export function constructRegExpObjectsFromRegExps(regexps: RegExpInput[]): RegEx
/**
* @ignore
*/
-export function createRequests(requestOptions: (string | RequestOptions)[], urlPatternObjects?: UrlPatternObject[]): Request[] {
+export function createRequests(
+ requestOptions: (string | RequestOptions)[],
+ urlPatternObjects?: UrlPatternObject[],
+ excludePatternObjects: UrlPatternObject[] = [],
+): Request[] {
if (!urlPatternObjects || !urlPatternObjects.length) {
return requestOptions
.map((opts) => new Request(typeof opts === 'string' ? { url: opts } : opts));
@@ -137,6 +141,21 @@ export function createRequests(requestOptions: (string | RequestOptions)[], urlP
for (const opts of requestOptions) {
const urlToMatch = typeof opts === 'string' ? opts : opts.url;
+ let isExcluded = false;
+ for (const excludePatternObject of excludePatternObjects) {
+ const { regexp, glob } = excludePatternObject;
+
+ if (
+ (regexp && urlToMatch.match(regexp)) || // eslint-disable-line
+ (glob && minimatch(urlToMatch, glob, { nocase: true }))
+ ) {
+ isExcluded = true;
+ break;
+ }
+ }
+
+ if (isExcluded) continue;
+
for (const urlPatternObject of urlPatternObjects) {
const { regexp, glob, ...requestRegExpOptions } = urlPatternObject;
if (
diff --git a/packages/core/test/enqueue_links/user-provided-patterns-with-enqueue-strategy.test.ts b/packages/core/test/enqueue_links/user-provided-patterns-with-enqueue-strategy.test.ts
index 33651fca7..e5f933f68 100644
--- a/packages/core/test/enqueue_links/user-provided-patterns-with-enqueue-strategy.test.ts
+++ b/packages/core/test/enqueue_links/user-provided-patterns-with-enqueue-strategy.test.ts
@@ -120,4 +120,24 @@ describe('enqueueLinks() - combining user patterns with enqueue strategies', ()
expect(enqueued[0].url).toBe('https://example.com/a/b/first');
expect(enqueued[1].url).toBe('https://example.com/a/b/third');
});
+
+ test('works with globs and exclude', async () => {
+ const { enqueued, requestQueue } = getMockRequestQueue();
+
+ const globs = ['**/first'];
+ const exclude = ['**/first'];
+
+ await cheerioCrawlerEnqueueLinks({
+ options: {
+ selector: '.click',
+ globs,
+ exclude,
+ },
+ $,
+ requestQueue,
+ originalRequestUrl: 'https://example.com',
+ });
+
+ expect(enqueued).toHaveLength(0);
+ });
});
diff --git a/test/core/enqueue_links/enqueue_links.test.ts b/test/core/enqueue_links/enqueue_links.test.ts
index 69fb03f0d..5f2e0270d 100644
--- a/test/core/enqueue_links/enqueue_links.test.ts
+++ b/test/core/enqueue_links/enqueue_links.test.ts
@@ -195,6 +195,100 @@ describe('enqueueLinks()', () => {
expect(enqueued[2].userData).toEqual({ label: 'COOL' });
});
+ test('works with exclude glob', async () => {
+ const enqueued: (Request | RequestOptions)[] = [];
+ const requestQueue = new RequestQueue({ id: 'xxx', client: apifyClient });
+
+ // @ts-expect-error Override method for testing
+ requestQueue.addRequests = async (request) => {
+ enqueued.push(...request);
+ };
+ const globs = [
+ 'https://example.com/**/*',
+ { glob: '?(http|https)://cool.com/', method: 'POST' as const },
+ ];
+
+ const exclude = ['**/first'];
+
+ await browserCrawlerEnqueueLinks({
+ options: {
+ selector: '.click',
+ label: 'COOL',
+ globs,
+ exclude,
+ transformRequestFunction: (request) => {
+ if (request.url.match(/example\.com\/a\/b\/third/)) {
+ request.method = 'OPTIONS';
+ }
+ return request;
+ },
+ },
+ page,
+ requestQueue,
+ originalRequestUrl: 'https://example.com',
+ });
+
+ expect(enqueued).toHaveLength(2);
+
+ expect(enqueued[0].url).not.toBe('https://example.com/a/b/first');
+ expect(enqueued[1].url).not.toBe('https://example.com/a/b/first');
+
+ expect(enqueued[0].url).toBe('https://example.com/a/b/third');
+ expect(enqueued[0].method).toBe('OPTIONS');
+ expect(enqueued[0].userData).toEqual({ label: 'COOL' });
+
+ expect(enqueued[1].url).toBe('http://cool.com/');
+ expect(enqueued[1].method).toBe('POST');
+ expect(enqueued[1].userData).toEqual({ label: 'COOL' });
+ });
+
+ test('works with exclude regexp', async () => {
+ const enqueued: (Request | RequestOptions)[] = [];
+ const requestQueue = new RequestQueue({ id: 'xxx', client: apifyClient });
+
+ // @ts-expect-error Override method for testing
+ requestQueue.addRequests = async (request) => {
+ enqueued.push(...request);
+ };
+ const globs = [
+ 'https://example.com/**/*',
+ { glob: '?(http|https)://cool.com/', method: 'POST' as const },
+ ];
+
+ const exclude = [/first/];
+
+ await browserCrawlerEnqueueLinks({
+ options: {
+ selector: '.click',
+ label: 'COOL',
+ globs,
+ exclude,
+ transformRequestFunction: (request) => {
+ if (request.url.match(/example\.com\/a\/b\/third/)) {
+ request.method = 'OPTIONS';
+ }
+ return request;
+ },
+ },
+ page,
+ requestQueue,
+ originalRequestUrl: 'https://example.com',
+ });
+
+ expect(enqueued).toHaveLength(2);
+
+ expect(enqueued[0].url).not.toBe('https://example.com/a/b/first');
+ expect(enqueued[1].url).not.toBe('https://example.com/a/b/first');
+
+ expect(enqueued[0].url).toBe('https://example.com/a/b/third');
+ expect(enqueued[0].method).toBe('OPTIONS');
+ expect(enqueued[0].userData).toEqual({ label: 'COOL' });
+
+ expect(enqueued[1].url).toBe('http://cool.com/');
+ expect(enqueued[1].method).toBe('POST');
+ expect(enqueued[1].userData).toEqual({ label: 'COOL' });
+ });
+
test('works with pseudoUrls', async () => {
const enqueued: (Request | RequestOptions)[] = [];
const requestQueue = new RequestQueue({ id: 'xxx', client: apifyClient });