Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

fix(basic): compile include/exclude patterns once per `addRequests` call (#4350)

Luigi Ruocco6df2aa61e2b0
diff --git a/packages/basic-crawler/src/internals/enqueue-links/shared.ts b/packages/basic-crawler/src/internals/enqueue-links/shared.ts
index e88fce0c7..bd4e26389 100644
--- a/packages/basic-crawler/src/internals/enqueue-links/shared.ts
+++ b/packages/basic-crawler/src/internals/enqueue-links/shared.ts
@@ -17,7 +17,7 @@ const MAX_ENQUEUE_LINKS_CACHE_SIZE = 1000;
* each glob/regexp is only compiled once and then kept in a cache.
* @ignore
*/
-const enqueueLinksPatternCache = new Map();
+const enqueueLinksPatternCache = new Map<UrlPatternInput, UrlPatternObject>();
export interface GlobObject {
glob: string;
@@ -79,14 +79,10 @@ export function createSkippedRequestArgs(
/**
* @ignore
*/
-export function updateEnqueueLinksPatternCache(
- item: GlobInput | RegExpInput,
- pattern: RegExpObject | GlobObject,
-): void {
+export function updateEnqueueLinksPatternCache(item: UrlPatternInput, pattern: UrlPatternObject): void {
enqueueLinksPatternCache.set(item, pattern);
if (enqueueLinksPatternCache.size > MAX_ENQUEUE_LINKS_CACHE_SIZE) {
- const key = enqueueLinksPatternCache.keys().next().value;
- enqueueLinksPatternCache.delete(key);
+ enqueueLinksPatternCache.delete(enqueueLinksPatternCache.keys().next().value!);
}
}
@@ -95,7 +91,7 @@ export function updateEnqueueLinksPatternCache(
* to construct Glob objects from Glob pattern strings.
* @ignore
*/
-export function constructGlobObjectsFromGlobs(globs: readonly GlobInput[]): GlobObject[] {
+export function constructGlobObjectsFromGlobs(globs: readonly GlobInput[]): (GlobObject & UrlPatternObject)[] {
return globs
.filter((glob) => {
// Skip possibly nullish, empty strings
@@ -114,15 +110,12 @@ export function constructGlobObjectsFromGlobs(globs: readonly GlobInput[]): Glob
return false;
})
.map((item) => {
- // Get glob object from cache.
- let globObject = enqueueLinksPatternCache.get(item);
- if (globObject) return globObject;
-
- if (typeof item === 'string') {
- globObject = { glob: validateGlobPattern(item) };
- } else {
- globObject = { glob: validateGlobPattern(item.glob) };
- }
+ const cached = enqueueLinksPatternCache.get(item);
+ if (cached) return cached as GlobObject & UrlPatternObject;
+
+ const glob = validateGlobPattern(typeof item === 'string' ? item : item.glob);
+ const minimatch = new Minimatch(glob, { nocase: true });
+ const globObject = { glob, match: (url: string) => minimatch.match(url) };
updateEnqueueLinksPatternCache(item, globObject);
@@ -145,17 +138,15 @@ export function validateGlobPattern(glob: string): string {
* to check RegExps input and return valid RegExps.
* @ignore
*/
-export function constructRegExpObjectsFromRegExps(regexps: readonly RegExpInput[]): RegExpObject[] {
+export function constructRegExpObjectsFromRegExps(
+ regexps: readonly RegExpInput[],
+): (RegExpObject & UrlPatternObject)[] {
return regexps.map((item) => {
- // Get regexp object from cache.
- let regexpObject = enqueueLinksPatternCache.get(item);
- if (regexpObject) return regexpObject;
-
- if (item instanceof RegExp) {
- regexpObject = { regexp: item };
- } else {
- regexpObject = { regexp: item.regexp };
- }
+ const cached = enqueueLinksPatternCache.get(item);
+ if (cached) return cached as RegExpObject & UrlPatternObject;
+
+ const regexp = item instanceof RegExp ? item : item.regexp;
+ const regexpObject = { regexp, match: (url: string) => regexp.test(url) };
updateEnqueueLinksPatternCache(item, regexpObject);
@@ -171,6 +162,7 @@ export function constructRegExpObjectsFromRegExps(regexps: readonly RegExpInput[
export interface UrlPatternObject {
glob?: string;
regexp?: RegExp;
+ match: (url: string) => boolean;
}
/**
@@ -205,23 +197,20 @@ export function filterRequestOptionsByPatterns(
strategy?: EnqueueStrategyOption,
onSkippedRequestOptions?: (options: RequestOptions) => void,
): RequestOptions[] {
- const excludeMatchers = excludePatterns.map(createPatternObjectMatcher);
- const includeMatchers = includePatterns?.length ? includePatterns.map(createPatternObjectMatcher) : undefined;
-
return requestOptions
.filter((opts) => {
- const matchesExclude = excludeMatchers.some(({ match }) => match(opts.url));
+ const matchesExclude = excludePatterns.some(({ match }) => match(opts.url));
if (matchesExclude) {
onSkippedRequestOptions?.(opts);
}
return !matchesExclude;
})
.map((opts) => {
- if (!includeMatchers) {
+ if (!includePatterns?.length) {
return { ...opts, enqueueStrategy: strategy };
}
- for (const { match } of includeMatchers) {
+ for (const { match } of includePatterns) {
if (match(opts.url)) {
return { ...opts, enqueueStrategy: strategy };
}
@@ -299,23 +288,6 @@ export function createRequestOptions(
});
}
-/**
- * @ignore
- */
-function createPatternObjectMatcher(urlPatternObject: UrlPatternObject) {
- const { regexp, glob } = urlPatternObject;
- let match: (url: string) => boolean;
- if (regexp) {
- match = (url: string) => regexp.test(url);
- } else if (glob) {
- const m = new Minimatch(glob, { nocase: true });
- match = (url: string) => m.match(url);
- } else {
- match = () => false;
- }
- return { match };
-}
-
/**
* Takes a {@apilink RequestOptions} object and changes its attributes in a desired way. This user-function is used
* by {@apilink enqueueLinks} to modify request options before they are converted to {@apilink Request} instances.
diff --git a/packages/basic-crawler/src/internals/sitemap-request-loader.ts b/packages/basic-crawler/src/internals/sitemap-request-loader.ts
index ad530e6b7..2bfe9163f 100644
--- a/packages/basic-crawler/src/internals/sitemap-request-loader.ts
+++ b/packages/basic-crawler/src/internals/sitemap-request-loader.ts
@@ -4,7 +4,6 @@ import type { BaseHttpClient } from '@crawlee/http-client';
import type { ParseSitemapOptions } from '@crawlee/utils';
import { EnqueueStrategy } from '@crawlee/utils';
import { parseArgument, parseSitemap, schemas } from '@crawlee/utils/internal';
-import { minimatch } from 'minimatch';
import type { RequiredDeep } from 'type-fest';
import { z } from 'zod';
@@ -271,22 +270,6 @@ export class SitemapRequestLoader implements IRequestLoader {
}).pause();
}
- /**
- * Returns a function that checks whether the provided pattern matches the closure URL.
- * @param url URL to be checked.
- * @returns A matcher function that checks whether the pattern matches the closure URL.
- */
- #matchesUrl(url: string): (patternObject: UrlPatternObject) => boolean {
- return (patternObject) => {
- const { regexp, glob } = patternObject;
-
- const matchesRegex = (regexp && url.match(regexp)) || false;
- const matchesGlob = (glob && minimatch(url, glob, { nocase: true })) || false;
-
- return Boolean(matchesRegex || matchesGlob);
- };
- }
-
/**
* Checks whether the URL matches the `include` / `exclude` patterns provided in the `options`.
* @param url URL to be checked.
@@ -294,8 +277,8 @@ export class SitemapRequestLoader implements IRequestLoader {
*/
#isUrlMatchingPatterns(url: string): boolean {
return (
- !this.#urlExcludePatternObjects.some(this.#matchesUrl(url)) &&
- (this.#urlPatternObjects.length === 0 || this.#urlPatternObjects.some(this.#matchesUrl(url)))
+ !this.#urlExcludePatternObjects.some(({ match }) => match(url)) &&
+ (this.#urlPatternObjects.length === 0 || this.#urlPatternObjects.some(({ match }) => match(url)))
);
}
diff --git a/test/core/enqueue-links/shared.test.ts b/test/core/enqueue-links/shared.test.ts
index 576538378..de728cdf4 100644
--- a/test/core/enqueue-links/shared.test.ts
+++ b/test/core/enqueue-links/shared.test.ts
@@ -51,11 +51,13 @@ describe('Enqueue links shared functions', () => {
test('should cache items', () => {
const globs0 = constructGlobObjectsFromGlobs(['https://example.com/**/*']);
const globs1 = constructGlobObjectsFromGlobs(['https://example.com/**/*']);
- expect(globs0[0]).toEqual(globs1[0]);
+ expect(globs0[0]).toBe(globs1[0]);
- const regexps0 = constructRegExpObjectsFromRegExps([/^https:\/\/example\.com\/(\w|\/)+/]);
- const regexps1 = constructRegExpObjectsFromRegExps([/^https:\/\/example\.com\/(\w|\/)+/]);
- expect(regexps0[0]).toEqual(regexps1[0]);
+ // RegExps are cached by instance, so the same one has to be passed twice
+ const regexp = /^https:\/\/example\.com\/(\w|\/)+/;
+ const regexps0 = constructRegExpObjectsFromRegExps([regexp]);
+ const regexps1 = constructRegExpObjectsFromRegExps([regexp]);
+ expect(regexps0[0]).toBe(regexps1[0]);
});
});