Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
fix: `createRequests` works correctly with `exclude` (and nothing else) (#2321)
Jindřich Bär048db0964a57
diff --git a/packages/core/src/enqueue_links/shared.ts b/packages/core/src/enqueue_links/shared.tsindex ee14d925c..eb862631b 100644--- a/packages/core/src/enqueue_links/shared.ts+++ b/packages/core/src/enqueue_links/shared.ts@@ -153,49 +153,40 @@ export function createRequests(excludePatternObjects: UrlPatternObject[] = [],strategy?: EnqueueLinksOptions['strategy'],): Request[] {- if (!urlPatternObjects || !urlPatternObjects.length) {- return requestOptions- .map((opts) => new Request(typeof opts === 'string' ? { url: opts, enqueueStrategy: strategy } : { ...opts }));- }-- const requests: Request[] = [];-- for (const opts of requestOptions) {- const urlToMatch = typeof opts === 'string' ? opts : opts.url;-- let isExcluded = false;- for (const excludePatternObject of excludePatternObjects) {- const { regexp, glob } = excludePatternObject;-- if (- (regexp && urlToMatch.match(regexp)) || // eslint-disable-line- (glob && minimatch(urlToMatch, glob, { nocase: true }))- ) {- isExcluded = true;- break;+ return requestOptions+ .map((opts) => ({ url: typeof opts === 'string' ? opts : opts.url, opts }))+ .filter(({ url }) => {+ return !excludePatternObjects.some((excludePatternObject) => {+ const { regexp, glob } = excludePatternObject;+ return (+ (regexp && url.match(regexp)) || // eslint-disable-line+ (glob && minimatch(url, glob, { nocase: true }))+ );+ });+ })+ .map(({ url, opts }) => {+ if (!urlPatternObjects || !urlPatternObjects.length) {+ return new Request(typeof opts === 'string' ? { url: opts, enqueueStrategy: strategy } : { ...opts });}- }-- if (isExcluded) continue;-- for (const urlPatternObject of urlPatternObjects) {- const { regexp, glob, ...requestRegExpOptions } = urlPatternObject;- if (- (regexp && urlToMatch.match(regexp)) || // eslint-disable-line- (glob && minimatch(urlToMatch, glob, { nocase: true }))- ) {- const request = typeof opts === 'string'- ? { url: opts, ...requestRegExpOptions, enqueueStrategy: strategy }- : { ...opts, ...requestRegExpOptions, enqueueStrategy: strategy };- requests.push(new Request(request));- // Stop checking other patterns for this request option as it was already matched- break;+ for (const urlPatternObject of urlPatternObjects) {+ const { regexp, glob, ...requestRegExpOptions } = urlPatternObject;+ if (+ (regexp && url.match(regexp)) || // eslint-disable-line+ (glob && minimatch(url, glob, { nocase: true }))+ ) {+ const request = typeof opts === 'string'+ ? { url: opts, ...requestRegExpOptions, enqueueStrategy: strategy }+ : { ...opts, ...requestRegExpOptions, enqueueStrategy: strategy };++ return new Request(request);+ }}- }- }- return requests;+ // didn't match any positive pattern+ return null;+ })+ .filter((request) => request) as Request[];}export function filterRequestsByPatterns(requests: Request[], patterns?: UrlPatternObject[]): Request[] {diff --git a/packages/core/test/enqueue_links/user-provided-patterns-with-enqueue-strategy.test.ts b/packages/core/test/enqueue_links/user-provided-patterns-with-enqueue-strategy.test.tsindex c5c4f311c..5ee6771f2 100644--- a/packages/core/test/enqueue_links/user-provided-patterns-with-enqueue-strategy.test.ts+++ b/packages/core/test/enqueue_links/user-provided-patterns-with-enqueue-strategy.test.ts@@ -140,4 +140,25 @@ describe('enqueueLinks() - combining user patterns with enqueue strategies', ()expect(enqueued).toHaveLength(0);});++ test('works with exclude only', async () => {+ const { enqueued, requestQueue } = createRequestQueueMock();++ const exclude = ['**/second', '**/third', 'https://another.com/**'];++ await cheerioCrawlerEnqueueLinks({+ options: {+ selector: '.click',+ exclude,+ strategy: EnqueueStrategy.All,+ },+ $,+ requestQueue,+ originalRequestUrl: 'https://example.com',+ });++ expect(enqueued).toHaveLength(2);+ expect(enqueued[0].url).toBe('https://example.com/a/b/first');+ expect(enqueued[1].url).toBe('http://cool.com/');+ });});