Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

fix(enqueueLinks): ensure the enqueue strategy is respected alongside user patterns (#1509)

Vlad Frangu2b0eeed3c5b0
diff --git a/packages/core/src/enqueue_links/enqueue_links.ts b/packages/core/src/enqueue_links/enqueue_links.ts
index 68ea26e0e..ad72f0930 100644
--- a/packages/core/src/enqueue_links/enqueue_links.ts
+++ b/packages/core/src/enqueue_links/enqueue_links.ts
@@ -4,6 +4,7 @@ import log from '@apify/log';
import type { BatchAddRequestsResult, Dictionary } from '@crawlee/types';
import type { GlobInput, PseudoUrlInput, RegExpInput, RequestTransform, UrlPatternObject } from './shared';
import {
+ filterRequestsByPatterns,
constructGlobObjectsFromGlobs,
constructRegExpObjectsFromPseudoUrls,
constructRegExpObjectsFromRegExps,
@@ -221,6 +222,8 @@ export async function enqueueLinks(options: EnqueueLinksOptions): Promise<BatchA
options.strategy ??= EnqueueStrategy.SameHostname;
}
+ const enqueueStrategyPatterns: UrlPatternObject[] = [];
+
if (options.baseUrl) {
const url = new URL(options.baseUrl);
@@ -229,7 +232,7 @@ export async function enqueueLinks(options: EnqueueLinksOptions): Promise<BatchA
// We need to get the origin of the passed in domain in the event someone sets baseUrl
// to an url like https://example.com/deep/default/path and one of the found urls is an
// absolute relative path (/path/to/page)
- urlPatternObjects.push({ glob: `${url.origin}/**` });
+ enqueueStrategyPatterns.push({ glob: `${url.origin}/**` });
break;
case EnqueueStrategy.SameDomain: {
// Get the actual hostname from the base url
@@ -238,14 +241,14 @@ export async function enqueueLinks(options: EnqueueLinksOptions): Promise<BatchA
if (baseUrlHostname) {
// We have a hostname, so we can use it to match all links on the page that point to it and any subdomains of it
url.hostname = baseUrlHostname;
- urlPatternObjects.push(
+ enqueueStrategyPatterns.push(
{ glob: `${url.origin.replace(baseUrlHostname, `*.${baseUrlHostname}`)}/**` },
{ glob: `${url.origin}/**` },
);
} else {
// We don't have a hostname (can happen for ips for instance), so reproduce the same behavior
// as SameDomainAndSubdomain
- urlPatternObjects.push({ glob: `${url.origin}/**` });
+ enqueueStrategyPatterns.push({ glob: `${url.origin}/**` });
}
break;
@@ -262,7 +265,19 @@ export async function enqueueLinks(options: EnqueueLinksOptions): Promise<BatchA
requestOptions = requestOptions.map((request) => transformRequestFunction(request)).filter((r) => !!r) as RequestOptions[];
}
- let requests = createRequests(requestOptions, urlPatternObjects);
+ function createFilteredRequests() {
+ // No user provided patterns means we can skip an extra filtering step
+ if (urlPatternObjects.length === 0) {
+ return createRequests(requestOptions, enqueueStrategyPatterns);
+ }
+
+ // Generate requests based on the user patterns first
+ const generatedRequestsFromUserFilters = createRequests(requestOptions, urlPatternObjects);
+ // ...then filter them by the enqueue links strategy (making this an AND check)
+ return filterRequestsByPatterns(generatedRequestsFromUserFilters, enqueueStrategyPatterns);
+ }
+
+ let requests = createFilteredRequests();
if (limit) requests = requests.slice(0, limit);
return requestQueue.addRequests(requests);
diff --git a/packages/core/src/enqueue_links/shared.ts b/packages/core/src/enqueue_links/shared.ts
index e9132e752..8ff816ef9 100644
--- a/packages/core/src/enqueue_links/shared.ts
+++ b/packages/core/src/enqueue_links/shared.ts
@@ -153,6 +153,31 @@ export function createRequests(requestOptions: (string | RequestOptions)[], urlP
return requests;
}
+export function filterRequestsByPatterns(requests: Request[], patterns?: UrlPatternObject[]) {
+ if (!patterns?.length) {
+ return requests;
+ }
+
+ const filtered: Request[] = [];
+
+ for (const request of requests) {
+ for (const urlPatternObject of patterns) {
+ const { regexp, glob } = urlPatternObject;
+
+ if (
+ (regexp && request.url.match(regexp)) || // eslint-disable-line
+ (glob && minimatch(request.url, glob, { nocase: true }))
+ ) {
+ filtered.push(request);
+ // Break the pattern loop, as we already matched this request once
+ break;
+ }
+ }
+ }
+
+ return filtered;
+}
+
/**
* @ignore
*/
diff --git a/packages/core/test/enqueue_links/user-provided-patterns-with-enqueue-strategy.test.ts b/packages/core/test/enqueue_links/user-provided-patterns-with-enqueue-strategy.test.ts
new file mode 100644
index 000000000..33651fca7
--- /dev/null
+++ b/packages/core/test/enqueue_links/user-provided-patterns-with-enqueue-strategy.test.ts
@@ -0,0 +1,123 @@
+import log from '@apify/log';
+import { load } from 'cheerio';
+import type { CheerioRoot, Request, RequestOptions } from 'crawlee';
+import { Configuration, cheerioCrawlerEnqueueLinks, RequestQueue, EnqueueStrategy } from 'crawlee';
+
+const apifyClient = Configuration.getStorageClient();
+
+const HTML = `
+<html>
+ <head>
+ <title>Example</title>
+ </head>
+ <body>
+ <p>
+ The ships hung in the sky, much the <a class="click" href="https://example.com/a/b/first">way that</a> bricks don't.
+ </p>
+ <ul>
+ <li>These aren't the Droids you're looking for</li>
+ <li><a href="https://example.com/a/second">I'm sorry, Dave. I'm afraid I can't do that.</a></li>
+ <li><a class="click" href="https://example.com/a/b/third">I'm sorry, Dave. I'm afraid I can't do that.</a></li>
+ </ul>
+ <a class="click" href="https://another.com/a/first">The Greatest Science Fiction Quotes Of All Time</a>
+ <p>
+ Don't know, I don't know such stuff. I just do eyes, ju-, ju-, just eyes... just genetic design,
+ just eyes. You Nexus, huh? I design your <a class="click" href="http://cool.com/">eyes</a>.
+ </p>
+ <a href="/x/absolutepath">This is a relative link.</a>
+ <a href="y/relativepath">This is a relative link.</a>
+ <a href="//example.absolute.com/hello">This is a link to a different subdomain</a>
+ </body>
+</html>
+`;
+
+function getMockRequestQueue() {
+ const enqueued: (Request | RequestOptions)[] = [];
+
+ const requestQueue = new RequestQueue({ id: 'xxx', client: apifyClient });
+
+ // @ts-expect-error Override method for testing
+ requestQueue.addRequests = (requests) => {
+ enqueued.push(...requests);
+ };
+
+ return { enqueued, requestQueue };
+}
+
+describe('enqueueLinks() - combining user patterns with enqueue strategies', () => {
+ let ll: number;
+ beforeAll(() => {
+ ll = log.getLevel();
+ log.setLevel(log.LEVELS.ERROR);
+ });
+
+ afterAll(() => {
+ log.setLevel(ll);
+ });
+
+ let $: CheerioRoot;
+ beforeEach(() => {
+ $ = load(HTML);
+ });
+
+ test('works with globs and same domain strategy', async () => {
+ const { enqueued, requestQueue } = getMockRequestQueue();
+
+ const globs = ['**/first'];
+
+ await cheerioCrawlerEnqueueLinks({
+ options: {
+ selector: '.click',
+ globs,
+ strategy: EnqueueStrategy.SameDomain,
+ },
+ $,
+ requestQueue,
+ originalRequestUrl: 'https://example.com',
+ });
+
+ expect(enqueued).toHaveLength(1);
+
+ expect(enqueued[0].url).toBe('https://example.com/a/b/first');
+ });
+
+ test('works with globs and all domains strategy', async () => {
+ const { enqueued, requestQueue } = getMockRequestQueue();
+
+ const globs = ['**/first'];
+
+ await cheerioCrawlerEnqueueLinks({
+ options: {
+ selector: '.click',
+ globs,
+ strategy: EnqueueStrategy.All,
+ },
+ $,
+ requestQueue,
+ originalRequestUrl: 'https://example.com',
+ });
+
+ expect(enqueued).toHaveLength(2);
+
+ expect(enqueued[0].url).toBe('https://example.com/a/b/first');
+ expect(enqueued[1].url).toBe('https://another.com/a/first');
+ });
+
+ test('works with no user provided patterns but with same domain strategy', async () => {
+ const { enqueued, requestQueue } = getMockRequestQueue();
+
+ await cheerioCrawlerEnqueueLinks({
+ options: {
+ selector: '.click',
+ strategy: EnqueueStrategy.SameDomain,
+ },
+ $,
+ requestQueue,
+ originalRequestUrl: 'https://example.com',
+ });
+
+ expect(enqueued).toHaveLength(2);
+ expect(enqueued[0].url).toBe('https://example.com/a/b/first');
+ expect(enqueued[1].url).toBe('https://example.com/a/b/third');
+ });
+});