Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

feat(enqueueLinks): add SameOrigin strategy and relax protocol matching for the other strategies (#1748)

Vlad Frangu4ba982a909a3
diff --git a/packages/core/src/enqueue_links/enqueue_links.ts b/packages/core/src/enqueue_links/enqueue_links.ts
index 3e42c4672..3a2d6ab1d 100644
--- a/packages/core/src/enqueue_links/enqueue_links.ts
+++ b/packages/core/src/enqueue_links/enqueue_links.ts
@@ -115,11 +115,46 @@ export interface EnqueueLinksOptions extends RequestQueueOperationOptions {
/**
* The strategy to use when enqueueing the urls.
+ *
+ * Depending on the strategy you select, we will only check certain parts of the URLs found. Here is a diagram of each URL part and their name:
+ *
+ * ```md
+ * Protocol Domain
+ * ┌────┐ ┌─────────┐
+ * https://example.crawlee.dev/...
+ * │ └─────────────────┤
+ * │ Hostname │
+ * │ │
+ * └─────────────────────────┘
+ * Origin
+ *```
+ *
* @default EnqueueStrategy.SameHostname
*/
- strategy?: EnqueueStrategy | 'all' | 'same-domain' | 'same-hostname';
+ strategy?: EnqueueStrategy | 'all' | 'same-domain' | 'same-hostname' | 'same-origin';
}
+/**
+ * The different enqueueing strategies available.
+ *
+ * Depending on the strategy you select, we will only check certain parts of the URLs found. Here is a diagram of each URL part and their name:
+ *
+ * ```md
+ * Protocol Domain
+ * ┌────┐ ┌─────────┐
+ * https://example.crawlee.dev/...
+ * │ └─────────────────┤
+ * │ Hostname │
+ * │ │
+ * └─────────────────────────┘
+ * Origin
+ *```
+ *
+ * - The `Protocol` is usually `http` or `https`
+ * - The `Domain` represents the path without any possible subdomains to a website. For example, `crawlee.dev` is the domain of `https://example.crawlee.dev/`
+ * - The `Hostname` is the full path to a website, including any subdomains. For example, `example.crawlee.dev` is the hostname of `https://example.crawlee.dev/`
+ * - The `Origin` is the combination of the `Protocol` and `Hostname`. For example, `https://example.crawlee.dev` is the origin of `https://example.crawlee.dev/`
+ */
export enum EnqueueStrategy {
/**
* Matches any URLs found
@@ -128,17 +163,30 @@ export enum EnqueueStrategy {
/**
* Matches any URLs that have the same hostname.
- * For example, `https://wow.example.com/hello` will be matched for a base url of `https://example.com/`, but
+ * For example, `https://wow.example.com/hello` will be matched for a base url of `https://wow.example.com/`, but
* `https://example.com/hello` will not be matched.
+ *
+ * > This strategy will match both `http` and `https` protocols regardless of the base URL protocol.
*/
SameHostname = 'same-hostname',
/**
- * Matches any URLs that have the same (sub-)domain as the base URL.
+ * Matches any URLs that have the same domain as the base URL.
* For example, `https://wow.an.example.com` and `https://example.com` will both be matched for a base url of
* `https://example.com`.
+ *
+ * > This strategy will match both `http` and `https` protocols regardless of the base URL protocol.
*/
SameDomain = 'same-domain',
+
+ /**
+ * Matches any URLs that have the same hostname and protocol.
+ * For example, `https://wow.example.com/hello` will be matched for a base url of `https://wow.example.com/`, but
+ * `http://wow.example.com/hello` will not be matched.
+ *
+ * > This strategy will ensure the protocol of the base URL is the same as the protocol of the URL to be enqueued.
+ */
+ SameOrigin = 'same-origin',
}
/**
@@ -196,11 +244,7 @@ export async function enqueueLinks(options: SetRequired<EnqueueLinksOptions, 're
ow.object.hasKeys('regexp'),
)),
transformRequestFunction: ow.optional.function,
- strategy: ow.optional.string.oneOf([
- EnqueueStrategy.All,
- EnqueueStrategy.SameHostname,
- EnqueueStrategy.SameDomain,
- ]),
+ strategy: ow.optional.string.oneOf(Object.values(EnqueueStrategy)),
}));
const {
@@ -243,7 +287,7 @@ export async function enqueueLinks(options: SetRequired<EnqueueLinksOptions, 're
// We need to get the origin of the passed in domain in the event someone sets baseUrl
// to an url like https://example.com/deep/default/path and one of the found urls is an
// absolute relative path (/path/to/page)
- enqueueStrategyPatterns.push({ glob: `${url.origin}/**` });
+ enqueueStrategyPatterns.push({ glob: ignoreHttpSchema(`${url.origin}/**`) });
break;
case EnqueueStrategy.SameDomain: {
// Get the actual hostname from the base url
@@ -253,17 +297,22 @@ export async function enqueueLinks(options: SetRequired<EnqueueLinksOptions, 're
// We have a hostname, so we can use it to match all links on the page that point to it and any subdomains of it
url.hostname = baseUrlHostname;
enqueueStrategyPatterns.push(
- { glob: `${url.origin.replace(baseUrlHostname, `*.${baseUrlHostname}`)}/**` },
- { glob: `${url.origin}/**` },
+ { glob: ignoreHttpSchema(`${url.origin.replace(baseUrlHostname, `*.${baseUrlHostname}`)}/**`) },
+ { glob: ignoreHttpSchema(`${url.origin}/**`) },
);
} else {
// We don't have a hostname (can happen for ips for instance), so reproduce the same behavior
// as SameDomainAndSubdomain
- enqueueStrategyPatterns.push({ glob: `${url.origin}/**` });
+ enqueueStrategyPatterns.push({ glob: ignoreHttpSchema(`${url.origin}/**`) });
}
break;
}
+ case EnqueueStrategy.SameOrigin: {
+ // The same behavior as SameHostname, but respecting the protocol of the URL
+ enqueueStrategyPatterns.push({ glob: `${url.origin}/**` });
+ break;
+ }
case EnqueueStrategy.All:
default:
break;
@@ -349,3 +398,10 @@ export interface ResolveBaseUrl {
originalRequestUrl: string;
finalRequestUrl?: string;
}
+
+/**
+ * Internal function that changes the enqueue globs to match both http and https
+ */
+function ignoreHttpSchema(pattern: string): string {
+ return pattern.replace(/^(https?):\/\//, 'http{s,}://');
+}
diff --git a/packages/core/test/enqueue_links/protocol-matching-based-on-strategy.test.ts b/packages/core/test/enqueue_links/protocol-matching-based-on-strategy.test.ts
new file mode 100644
index 000000000..97c0ed5c0
--- /dev/null
+++ b/packages/core/test/enqueue_links/protocol-matching-based-on-strategy.test.ts
@@ -0,0 +1,94 @@
+import log from '@apify/log';
+import { load } from 'cheerio';
+import type { CheerioRoot, Request, RequestOptions } from 'crawlee';
+import { EnqueueStrategy, Configuration, cheerioCrawlerEnqueueLinks, RequestQueue } from 'crawlee';
+
+const apifyClient = Configuration.getStorageClient();
+
+const HTML = `
+<html>
+ <head>
+ <title>Example</title>
+ </head>
+ <body>
+ <ul>
+ <li><a class="first" href="https://example.com/first">I'm sorry, Dave. I'm afraid I can't do that.</a></li>
+ <li><a class="second" href="http://example.com/second">I'm sorry, Dave. I'm afraid I can't do that.</a></li>
+ </ul>
+ </body>
+</html>
+`;
+
+function getMockRequestQueue() {
+ const enqueued: (Request | RequestOptions)[] = [];
+
+ const requestQueue = new RequestQueue({ id: 'xxx', client: apifyClient });
+
+ // @ts-expect-error Override method for testing
+ requestQueue.addRequests = (requests) => {
+ enqueued.push(...requests);
+ };
+
+ return { enqueued, requestQueue };
+}
+
+describe('enqueueLinks() - matching and ignoring http/https protocol differences', () => {
+ let ll: number;
+ beforeAll(() => {
+ ll = log.getLevel();
+ log.setLevel(log.LEVELS.ERROR);
+ });
+
+ afterAll(() => {
+ log.setLevel(ll);
+ });
+
+ let $: CheerioRoot;
+ beforeEach(() => {
+ $ = load(HTML);
+ });
+
+ test('SameHostname should ignore protocol difference', async () => {
+ const { enqueued, requestQueue } = getMockRequestQueue();
+
+ await cheerioCrawlerEnqueueLinks({
+ options: { selector: 'a', strategy: EnqueueStrategy.SameHostname },
+ $,
+ requestQueue,
+ originalRequestUrl: 'https://example.com',
+ });
+
+ expect(enqueued).toHaveLength(2);
+ expect(enqueued[0].url).toBe('https://example.com/first');
+ expect(enqueued[1].url).toBe('http://example.com/second');
+ });
+
+ test('SameDomain should ignore protocol difference', async () => {
+ const { enqueued, requestQueue } = getMockRequestQueue();
+
+ await cheerioCrawlerEnqueueLinks({
+ options: { selector: 'a', strategy: EnqueueStrategy.SameDomain },
+ $,
+ requestQueue,
+ originalRequestUrl: 'http://example.com',
+ });
+
+ expect(enqueued).toHaveLength(2);
+ expect(enqueued[0].url).toBe('https://example.com/first');
+ expect(enqueued[1].url).toBe('http://example.com/second');
+ });
+
+ test('SameOrigin should respect protocol', async () => {
+ const { enqueued, requestQueue } = getMockRequestQueue();
+
+ await cheerioCrawlerEnqueueLinks({
+ options: { selector: 'a', strategy: EnqueueStrategy.SameOrigin },
+ $,
+ requestQueue,
+ originalRequestUrl: 'https://example.com',
+ });
+
+ expect(enqueued).toHaveLength(1);
+ expect(enqueued[0].url).toBe('https://example.com/first');
+ });
+});
diff --git a/test/core/enqueue_links/enqueue_links.test.ts b/test/core/enqueue_links/enqueue_links.test.ts
index 29e20116a..0e6d3465d 100644
--- a/test/core/enqueue_links/enqueue_links.test.ts
+++ b/test/core/enqueue_links/enqueue_links.test.ts
@@ -399,7 +399,7 @@ describe('enqueueLinks()', () => {
expect(enqueued[1].userData).toEqual({});
});
- test('correctly resolves relative URLs with the strategy of same-hostname', async () => {
+ test('correctly resolves relative URLs with the strategy of same-domain', async () => {
const enqueued: (Request | RequestOptions)[] = [];
const requestQueue = new RequestQueue({ id: 'xxx', client: apifyClient });
@@ -874,7 +874,7 @@ describe('enqueueLinks()', () => {
expect(enqueued[1].userData).toEqual({});
});
- test('correctly resolves relative URLs with the strategy of same-hostname', async () => {
+ test('correctly resolves relative URLs with the strategy of same-domain', async () => {
const enqueued: (Request | RequestOptions)[] = [];
const requestQueue = new RequestQueue({ id: 'xxx', client: apifyClient });