Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
fix(enqueueLinks): prevent useless request creations when filtering by user patterns (#1510)
Vlad Frangucb8fe3664db1
diff --git a/packages/core/src/enqueue_links/shared.ts b/packages/core/src/enqueue_links/shared.tsindex 8ff816ef9..cd66e66bb 100644--- a/packages/core/src/enqueue_links/shared.ts+++ b/packages/core/src/enqueue_links/shared.ts@@ -134,10 +134,11 @@ export function createRequests(requestOptions: (string | RequestOptions)[], urlPconst requests: Request[] = [];- for (const urlPatternObject of urlPatternObjects) {- const { regexp, glob, ...requestRegExpOptions } = urlPatternObject;- for (const opts of requestOptions) {- const urlToMatch = typeof opts === 'string' ? opts : opts.url;+ for (const opts of requestOptions) {+ const urlToMatch = typeof opts === 'string' ? opts : opts.url;++ for (const urlPatternObject of urlPatternObjects) {+ const { regexp, glob, ...requestRegExpOptions } = urlPatternObject;if ((regexp && urlToMatch.match(regexp)) || // eslint-disable-line(glob && minimatch(urlToMatch, glob, { nocase: true }))@@ -146,6 +147,9 @@ export function createRequests(requestOptions: (string | RequestOptions)[], urlP? { url: opts, ...requestRegExpOptions }: { ...opts, ...requestRegExpOptions };requests.push(new Request(request));++ // Stop checking other patterns for this request option as it was already matched+ break;}}}