Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

perf: Improve glob performance by reusing minimatch objects (#3168)

Edgars Beigartse5632e270019
diff --git a/packages/core/src/enqueue_links/shared.ts b/packages/core/src/enqueue_links/shared.ts
index e7344b1ad..a75e5c277 100644
--- a/packages/core/src/enqueue_links/shared.ts
+++ b/packages/core/src/enqueue_links/shared.ts
@@ -1,7 +1,7 @@
import { URL } from 'node:url';
import type { Awaitable } from '@crawlee/types';
-import { minimatch } from 'minimatch';
+import { Minimatch } from 'minimatch';
import { purlToRegExp } from '@apify/pseudo_url';
@@ -176,13 +176,13 @@ export function createRequests(
strategy?: EnqueueLinksOptions['strategy'],
onSkippedUrl?: (url: string) => void,
): Request[] {
+ const excludePatternObjectMatchers = excludePatternObjects.map(createPatternObjectMatcher);
+ const urlPatternObjectMatchers = urlPatternObjects?.map(createPatternObjectMatcher);
+
return requestOptions
.map((opts) => ({ url: typeof opts === 'string' ? opts : opts.url, opts }))
.filter(({ url }) => {
- const matchesExcludePatterns = excludePatternObjects.some((excludePatternObject) => {
- const { regexp, glob } = excludePatternObject;
- return (regexp && url.match(regexp)) || (glob && minimatch(url, glob, { nocase: true }));
- });
+ const matchesExcludePatterns = excludePatternObjectMatchers.some(({ match }) => match(url));
if (matchesExcludePatterns) {
onSkippedUrl?.(url);
@@ -191,13 +191,13 @@ export function createRequests(
return !matchesExcludePatterns;
})
.map(({ url, opts }) => {
- if (!urlPatternObjects || !urlPatternObjects.length) {
+ if (!urlPatternObjectMatchers || !urlPatternObjectMatchers.length) {
return new Request(typeof opts === 'string' ? { url: opts, enqueueStrategy: strategy } : { ...opts });
}
- for (const urlPatternObject of urlPatternObjects) {
- const { regexp, glob, ...requestRegExpOptions } = urlPatternObject;
- if ((regexp && url.match(regexp)) || (glob && minimatch(url, glob, { nocase: true }))) {
+ for (const urlPatternObject of urlPatternObjectMatchers) {
+ const { match, glob, regexp, ...requestRegExpOptions } = urlPatternObject;
+ if (match(url)) {
const request =
typeof opts === 'string'
? { url: opts, ...requestRegExpOptions, enqueueStrategy: strategy }
@@ -224,12 +224,10 @@ export function filterRequestsByPatterns(
}
const filtered: Request[] = [];
+ const patternMatchers = patterns?.map(createPatternObjectMatcher);
for (const request of requests) {
- const matchingPattern = patterns.find(
- ({ regexp, glob }) =>
- (regexp && request.url.match(regexp)) || (glob && minimatch(request.url, glob, { nocase: true })),
- );
+ const matchingPattern = patternMatchers.find(({ match }) => match(request.url));
if (matchingPattern !== undefined) {
filtered.push(request);
@@ -280,6 +278,23 @@ export function createRequestOptions(
});
}
+/**
+ * @ignore
+ */
+function createPatternObjectMatcher(urlPatternObject: UrlPatternObject) {
+ const { regexp, glob } = urlPatternObject;
+ let match;
+ if (regexp) {
+ match = (url: string) => regexp.test(url);
+ } else if (glob) {
+ const m = new Minimatch(glob, { nocase: true });
+ match = (url: string) => m.match(url);
+ } else {
+ match = () => false;
+ }
+ return { ...urlPatternObject, match };
+}
+
/**
* Takes an Apify {@apilink RequestOptions} object and changes its attributes in a desired way. This user-function is used
* {@apilink enqueueLinks} to modify requests before enqueuing them.