Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

fix: respect `<base>` when enqueuing (#1936)

Martin Adámekaeef57231c84
diff --git a/packages/browser-crawler/src/internals/browser-crawler.ts b/packages/browser-crawler/src/internals/browser-crawler.ts
index 8cd603f3f..665dbc05e 100644
--- a/packages/browser-crawler/src/internals/browser-crawler.ts
+++ b/packages/browser-crawler/src/internals/browser-crawler.ts
@@ -1,29 +1,29 @@
import { addTimeoutToPromise, tryCancel } from '@apify/timeout';
import type {
+ Awaitable,
+ BasicCrawlerOptions,
CrawlingContext,
+ Dictionary,
+ EnqueueLinksOptions,
+ ErrorHandler,
ProxyConfiguration,
ProxyInfo,
+ RequestHandler,
RequestQueue,
Session,
- BasicCrawlerOptions,
- Awaitable,
- Dictionary,
- RequestHandler,
- ErrorHandler,
- EnqueueLinksOptions,
} from '@crawlee/basic';
import {
+ BASIC_CRAWLER_TIMEOUT_BUFFER_SECS,
+ BasicCrawler,
+ Configuration,
cookieStringToToughCookie,
enqueueLinks,
EVENT_SESSION_RETIRED,
handleRequestTimeout,
- validators,
- resolveBaseUrlForEnqueueLinksFiltering,
- Configuration,
- BASIC_CRAWLER_TIMEOUT_BUFFER_SECS,
- BasicCrawler,
- RequestState,
tryAbsoluteURL,
+ RequestState,
+ resolveBaseUrlForEnqueueLinksFiltering,
+ validators,
} from '@crawlee/basic';
import type {
BrowserController,
@@ -714,15 +714,21 @@ export async function browserCrawlerEnqueueLinks({
* @ignore
*/
// eslint-disable-next-line @typescript-eslint/ban-types
-async function extractUrlsFromPage(page: { $$eval: Function }, selector: string, baseUrl?: string): Promise<string[]> {
+async function extractUrlsFromPage(page: { $$eval: Function }, selector: string, baseUrl: string): Promise<string[]> {
const urls = await page.$$eval(selector, (linkEls: HTMLLinkElement[]) => linkEls.map((link) => link.getAttribute('href')).filter((href) => !!href)) ?? [];
+ const [base] = await page.$$eval('base', (els: HTMLLinkElement[]) => els.map((el) => el.getAttribute('href')));
+ const absoluteBaseUrl = base && tryAbsoluteURL(base, baseUrl);
+
+ if (absoluteBaseUrl) {
+ baseUrl = absoluteBaseUrl;
+ }
return urls.map((href: string) => {
// Throw a meaningful error when only a relative URL would be extracted instead of waiting for the Request to fail later.
const isHrefAbsolute = /^[a-z][a-z0-9+.-]*:/.test(href); // Grabbed this in 'is-absolute-url' package.
if (!isHrefAbsolute && !baseUrl) {
throw new Error(`An extracted URL: ${href} is relative and options.baseUrl is not set. `
- + 'Use options.baseUrl in enqueueLinks() to automatically resolve relative URLs.');
+ + 'Use options.baseUrl in enqueueLinks() to automatically resolve relative URLs.');
}
return baseUrl
diff --git a/packages/cheerio-crawler/src/internals/cheerio-crawler.ts b/packages/cheerio-crawler/src/internals/cheerio-crawler.ts
index 472d6c207..9385b25f1 100644
--- a/packages/cheerio-crawler/src/internals/cheerio-crawler.ts
+++ b/packages/cheerio-crawler/src/internals/cheerio-crawler.ts
@@ -234,7 +234,14 @@ export async function cheerioCrawlerEnqueueLinks({ options, $, requestQueue, ori
* Extracts URLs from a given Cheerio object.
* @ignore
*/
-function extractUrlsFromCheerio($: cheerio.CheerioAPI, selector: string, baseUrl?: string): string[] {
+function extractUrlsFromCheerio($: cheerio.CheerioAPI, selector: string, baseUrl: string): string[] {
+ const base = $('base').attr('href');
+ const absoluteBaseUrl = base && tryAbsoluteURL(base, baseUrl);
+
+ if (absoluteBaseUrl) {
+ baseUrl = absoluteBaseUrl;
+ }
+
return $(selector)
.map((_i, el) => $(el).attr('href'))
.get()
diff --git a/test/e2e/cheerio-enqueue-links-base/actor/.actor/actor.json b/test/e2e/cheerio-enqueue-links-base/actor/.actor/actor.json
new file mode 100644
index 000000000..0087602ef
--- /dev/null
+++ b/test/e2e/cheerio-enqueue-links-base/actor/.actor/actor.json
@@ -0,0 +1,7 @@
+{
+ "actorSpecification": 1,
+ "name": "test-cheerio-enqueue-links-base",
+ "version": "0.0",
+ "buildTag": "latest",
+ "env": null
+}
diff --git a/test/e2e/cheerio-enqueue-links-base/actor/.gitignore b/test/e2e/cheerio-enqueue-links-base/actor/.gitignore
new file mode 100644
index 000000000..ced7cbfc5
--- /dev/null
+++ b/test/e2e/cheerio-enqueue-links-base/actor/.gitignore
@@ -0,0 +1,7 @@
+.idea
+.DS_Store
+node_modules
+package-lock.json
+apify_storage
+crawlee_storage
+storage
diff --git a/test/e2e/cheerio-enqueue-links-base/actor/Dockerfile b/test/e2e/cheerio-enqueue-links-base/actor/Dockerfile
new file mode 100644
index 000000000..58806cec5
--- /dev/null
+++ b/test/e2e/cheerio-enqueue-links-base/actor/Dockerfile
@@ -0,0 +1,16 @@
+FROM apify/actor-node:18-beta
+
+COPY packages ./packages
+COPY package*.json ./
+
+RUN npm --quiet set progress=false \
+ && npm install --only=prod --no-optional \
+ && npm update \
+ && echo "Installed NPM packages:" \
+ && (npm list --only=prod --no-optional --all || true) \
+ && echo "Node.js version:" \
+ && node --version \
+ && echo "NPM version:" \
+ && npm --version
+
+COPY . ./
diff --git a/test/e2e/cheerio-enqueue-links-base/actor/main.js b/test/e2e/cheerio-enqueue-links-base/actor/main.js
new file mode 100644
index 000000000..514fa7934
--- /dev/null
+++ b/test/e2e/cheerio-enqueue-links-base/actor/main.js
@@ -0,0 +1,31 @@
+import { Actor } from 'apify';
+import { CheerioCrawler, Dataset } from '@crawlee/cheerio';
+
+const mainOptions = {
+ exit: Actor.isAtHome(),
+ storage: process.env.STORAGE_IMPLEMENTATION === 'LOCAL' ? new (await import('@apify/storage-local')).ApifyStorageLocal() : undefined,
+};
+
+await Actor.main(async () => {
+ const crawler = new CheerioCrawler({
+ maxRequestsPerCrawl: 30,
+ proxyConfiguration: await Actor.createProxyConfiguration(),
+ async requestHandler({ $, enqueueLinks, request, log }) {
+ const { url, loadedUrl } = request;
+
+ const pageTitle = $('title').first().text();
+ log.info(`URL: ${url}; LOADED_URL: ${loadedUrl}; TITLE: ${pageTitle}`);
+ await Dataset.pushData({ url, loadedUrl, pageTitle });
+
+ await enqueueLinks({
+ globs: [
+ 'https://www.jamesallen.com/about-us/**',
+ 'https://www.jamesallen.com/terms-of-use/**',
+ 'https://www.jamesallen.com/guarantee/**',
+ ],
+ });
+ },
+ });
+
+ await crawler.run(['https://www.jamesallen.com/faq/']);
+}, mainOptions);
diff --git a/test/e2e/cheerio-enqueue-links-base/actor/package.json b/test/e2e/cheerio-enqueue-links-base/actor/package.json
new file mode 100644
index 000000000..9c4711b45
--- /dev/null
+++ b/test/e2e/cheerio-enqueue-links-base/actor/package.json
@@ -0,0 +1,29 @@
+{
+ "name": "test-cheerio-enqueue-links-base",
+ "version": "0.0.1",
+ "description": "Cheerio Crawler Test - Enqueue Links with <base href>",
+ "dependencies": {
+ "apify": "next",
+ "@apify/storage-local": "^2.1.0",
+ "@crawlee/basic": "file:./packages/basic-crawler",
+ "@crawlee/browser-pool": "file:./packages/browser-pool",
+ "@crawlee/http": "file:./packages/http-crawler",
+ "@crawlee/cheerio": "file:./packages/cheerio-crawler",
+ "@crawlee/core": "file:./packages/core",
+ "@crawlee/memory-storage": "file:./packages/memory-storage",
+ "@crawlee/types": "file:./packages/types",
+ "@crawlee/utils": "file:./packages/utils",
+ "deep-equal": "^2.0.5"
+ },
+ "overrides": {
+ "apify": {
+ "@crawlee/core": "file:./packages/core",
+ "@crawlee/utils": "file:./packages/utils"
+ }
+ },
+ "scripts": {
+ "start": "node main.js"
+ },
+ "type": "module",
+ "license": "ISC"
+}
diff --git a/test/e2e/cheerio-enqueue-links-base/test.mjs b/test/e2e/cheerio-enqueue-links-base/test.mjs
new file mode 100644
index 000000000..502745fdd
--- /dev/null
+++ b/test/e2e/cheerio-enqueue-links-base/test.mjs
@@ -0,0 +1,8 @@
+import { initialize, getActorTestDir, runActor, expect } from '../tools.mjs';
+
+const testActorDirname = getActorTestDir(import.meta.url);
+await initialize(testActorDirname);
+
+const { datasetItems } = await runActor(testActorDirname);
+
+await expect(datasetItems.length === 14, `Enqueueing respects <base href>`);
diff --git a/test/e2e/jsdom-react-ts/test.mjs b/test/e2e/jsdom-react-ts/test.mjs
index 93723ba2d..0b89623a5 100644
--- a/test/e2e/jsdom-react-ts/test.mjs
+++ b/test/e2e/jsdom-react-ts/test.mjs
@@ -1,4 +1,6 @@
-import { initialize, getActorTestDir, runActor, expect, validateDataset } from '../tools.mjs';
+import { initialize, getActorTestDir, runActor, expect, validateDataset, skipTest } from '../tools.mjs';
+
+await skipTest('target site no longer exists');
const testActorDirname = getActorTestDir(import.meta.url);
await initialize(testActorDirname);
diff --git a/test/e2e/playwright-chromium-experimental-containers/actor/Dockerfile b/test/e2e/playwright-chromium-experimental-containers/actor/Dockerfile
index 0be6b9d47..571491204 100644
--- a/test/e2e/playwright-chromium-experimental-containers/actor/Dockerfile
+++ b/test/e2e/playwright-chromium-experimental-containers/actor/Dockerfile
@@ -1,4 +1,4 @@
-FROM node:18-beta AS builder
+FROM node:18 AS builder
COPY /packages ./packages
COPY /package*.json ./
diff --git a/test/e2e/playwright-default/actor/Dockerfile b/test/e2e/playwright-default/actor/Dockerfile
index 0be6b9d47..571491204 100644
--- a/test/e2e/playwright-default/actor/Dockerfile
+++ b/test/e2e/playwright-default/actor/Dockerfile
@@ -1,4 +1,4 @@
-FROM node:18-beta AS builder
+FROM node:18 AS builder
COPY /packages ./packages
COPY /package*.json ./
diff --git a/test/e2e/playwright-enqueue-links-base/actor/.actor/actor.json b/test/e2e/playwright-enqueue-links-base/actor/.actor/actor.json
new file mode 100644
index 000000000..e40f74e8a
--- /dev/null
+++ b/test/e2e/playwright-enqueue-links-base/actor/.actor/actor.json
@@ -0,0 +1,7 @@
+{
+ "actorSpecification": 1,
+ "name": "test-playwright-enqueue-links-base",
+ "version": "0.0",
+ "buildTag": "latest",
+ "env": null
+}
diff --git a/test/e2e/playwright-enqueue-links-base/actor/.gitignore b/test/e2e/playwright-enqueue-links-base/actor/.gitignore
new file mode 100644
index 000000000..ced7cbfc5
--- /dev/null
+++ b/test/e2e/playwright-enqueue-links-base/actor/.gitignore
@@ -0,0 +1,7 @@
+.idea
+.DS_Store
+node_modules
+package-lock.json
+apify_storage
+crawlee_storage
+storage
diff --git a/test/e2e/playwright-enqueue-links-base/actor/Dockerfile b/test/e2e/playwright-enqueue-links-base/actor/Dockerfile
new file mode 100644
index 000000000..571491204
--- /dev/null
+++ b/test/e2e/playwright-enqueue-links-base/actor/Dockerfile
@@ -0,0 +1,23 @@
+FROM node:18 AS builder
+
+COPY /packages ./packages
+COPY /package*.json ./
+RUN npm --quiet set progress=false \
+ && npm install --only=prod --no-optional \
+ && npm update
+
+FROM apify/actor-node-playwright-chrome:18-beta
+
+RUN rm -r node_modules
+COPY --from=builder /node_modules ./node_modules
+COPY --from=builder /packages ./packages
+COPY --from=builder /package*.json ./
+COPY /.actor ./.actor
+COPY /main.js ./
+
+RUN echo "Installed NPM packages:" \
+ && (npm list --only=prod --no-optional --all || true) \
+ && echo "Node.js version:" \
+ && node --version \
+ && echo "NPM version:" \
+ && npm --version
diff --git a/test/e2e/playwright-enqueue-links-base/actor/main.js b/test/e2e/playwright-enqueue-links-base/actor/main.js
new file mode 100644
index 000000000..9db543080
--- /dev/null
+++ b/test/e2e/playwright-enqueue-links-base/actor/main.js
@@ -0,0 +1,32 @@
+import { Actor } from 'apify';
+import { PlaywrightCrawler, Dataset } from '@crawlee/playwright';
+
+const mainOptions = {
+ exit: Actor.isAtHome(),
+ storage: process.env.STORAGE_IMPLEMENTATION === 'LOCAL' ? new (await import('@apify/storage-local')).ApifyStorageLocal() : undefined,
+};
+
+await Actor.main(async () => {
+ const crawler = new PlaywrightCrawler({
+ maxRequestsPerCrawl: 30,
+ proxyConfiguration: await Actor.createProxyConfiguration(),
+ async requestHandler({ parseWithCheerio, enqueueLinks, request, log }) {
+ const { url, loadedUrl } = request;
+
+ const $ = await parseWithCheerio();
+ const pageTitle = $('title').first().text();
+ log.info(`URL: ${url}; LOADED_URL: ${loadedUrl}; TITLE: ${pageTitle}`);
+ await Dataset.pushData({ url, loadedUrl, pageTitle });
+
+ await enqueueLinks({
+ globs: [
+ 'https://www.jamesallen.com/about-us/**',
+ 'https://www.jamesallen.com/terms-of-use/**',
+ 'https://www.jamesallen.com/guarantee/**',
+ ],
+ });
+ },
+ });
+
+ await crawler.run(['https://www.jamesallen.com/faq/']);
+}, mainOptions);
diff --git a/test/e2e/playwright-enqueue-links-base/actor/package.json b/test/e2e/playwright-enqueue-links-base/actor/package.json
new file mode 100644
index 000000000..bae23adab
--- /dev/null
+++ b/test/e2e/playwright-enqueue-links-base/actor/package.json
@@ -0,0 +1,30 @@
+{
+ "name": "test-playwright-enqueue-links-base",
+ "version": "0.0.1",
+ "description": "Playwright Test - Enqueue Links with <base href>",
+ "dependencies": {
+ "apify": "next",
+ "@apify/storage-local": "^2.1.0",
+ "@crawlee/basic": "file:./packages/basic-crawler",
+ "@crawlee/browser": "file:./packages/browser-crawler",
+ "@crawlee/browser-pool": "file:./packages/browser-pool",
+ "@crawlee/core": "file:./packages/core",
+ "@crawlee/memory-storage": "file:./packages/memory-storage",
+ "@crawlee/playwright": "file:./packages/playwright-crawler",
+ "@crawlee/types": "file:./packages/types",
+ "@crawlee/utils": "file:./packages/utils",
+ "deep-equal": "^2.0.5",
+ "playwright": "*"
+ },
+ "overrides": {
+ "apify": {
+ "@crawlee/core": "file:./packages/core",
+ "@crawlee/utils": "file:./packages/utils"
+ }
+ },
+ "scripts": {
+ "start": "node main.js"
+ },
+ "type": "module",
+ "license": "ISC"
+}
diff --git a/test/e2e/playwright-enqueue-links-base/test.mjs b/test/e2e/playwright-enqueue-links-base/test.mjs
new file mode 100644
index 000000000..502745fdd
--- /dev/null
+++ b/test/e2e/playwright-enqueue-links-base/test.mjs
@@ -0,0 +1,8 @@
+import { initialize, getActorTestDir, runActor, expect } from '../tools.mjs';
+
+const testActorDirname = getActorTestDir(import.meta.url);
+await initialize(testActorDirname);
+
+const { datasetItems } = await runActor(testActorDirname);
+
+await expect(datasetItems.length === 14, `Enqueueing respects <base href>`);
diff --git a/test/e2e/playwright-enqueue-links/actor/Dockerfile b/test/e2e/playwright-enqueue-links/actor/Dockerfile
index 0be6b9d47..571491204 100644
--- a/test/e2e/playwright-enqueue-links/actor/Dockerfile
+++ b/test/e2e/playwright-enqueue-links/actor/Dockerfile
@@ -1,4 +1,4 @@
-FROM node:18-beta AS builder
+FROM node:18 AS builder
COPY /packages ./packages
COPY /package*.json ./
diff --git a/test/e2e/playwright-firefox-experimental-containers/actor/Dockerfile b/test/e2e/playwright-firefox-experimental-containers/actor/Dockerfile
index dd42e7f17..47d04fac5 100644
--- a/test/e2e/playwright-firefox-experimental-containers/actor/Dockerfile
+++ b/test/e2e/playwright-firefox-experimental-containers/actor/Dockerfile
@@ -1,4 +1,4 @@
-FROM node:18-beta AS builder
+FROM node:18 AS builder
COPY /packages ./packages
COPY /package*.json ./
diff --git a/test/e2e/playwright-initial-cookies/actor/Dockerfile b/test/e2e/playwright-initial-cookies/actor/Dockerfile
index 0be6b9d47..571491204 100644
--- a/test/e2e/playwright-initial-cookies/actor/Dockerfile
+++ b/test/e2e/playwright-initial-cookies/actor/Dockerfile
@@ -1,4 +1,4 @@
-FROM node:18-beta AS builder
+FROM node:18 AS builder
COPY /packages ./packages
COPY /package*.json ./
diff --git a/test/e2e/playwright-multi-run/actor/Dockerfile b/test/e2e/playwright-multi-run/actor/Dockerfile
index 0be6b9d47..571491204 100644
--- a/test/e2e/playwright-multi-run/actor/Dockerfile
+++ b/test/e2e/playwright-multi-run/actor/Dockerfile
@@ -1,4 +1,4 @@
-FROM node:18-beta AS builder
+FROM node:18 AS builder
COPY /packages ./packages
COPY /package*.json ./
diff --git a/test/e2e/proxy-rotation/actor/Dockerfile b/test/e2e/proxy-rotation/actor/Dockerfile
index 505d02a8d..fba9ec484 100644
--- a/test/e2e/proxy-rotation/actor/Dockerfile
+++ b/test/e2e/proxy-rotation/actor/Dockerfile
@@ -1,4 +1,4 @@
-FROM node:18-beta AS builder
+FROM node:18 AS builder
COPY /packages ./packages
COPY /package*.json ./
diff --git a/test/e2e/puppeteer-default/actor/Dockerfile b/test/e2e/puppeteer-default/actor/Dockerfile
index 505d02a8d..fba9ec484 100644
--- a/test/e2e/puppeteer-default/actor/Dockerfile
+++ b/test/e2e/puppeteer-default/actor/Dockerfile
@@ -1,4 +1,4 @@
-FROM node:18-beta AS builder
+FROM node:18 AS builder
COPY /packages ./packages
COPY /package*.json ./
diff --git a/test/e2e/puppeteer-enqueue-links/actor/Dockerfile b/test/e2e/puppeteer-enqueue-links/actor/Dockerfile
index 505d02a8d..fba9ec484 100644
--- a/test/e2e/puppeteer-enqueue-links/actor/Dockerfile
+++ b/test/e2e/puppeteer-enqueue-links/actor/Dockerfile
@@ -1,4 +1,4 @@
-FROM node:18-beta AS builder
+FROM node:18 AS builder
COPY /packages ./packages
COPY /package*.json ./
diff --git a/test/e2e/puppeteer-ignore-ssl-errors/actor/Dockerfile b/test/e2e/puppeteer-ignore-ssl-errors/actor/Dockerfile
index 505d02a8d..fba9ec484 100644
--- a/test/e2e/puppeteer-ignore-ssl-errors/actor/Dockerfile
+++ b/test/e2e/puppeteer-ignore-ssl-errors/actor/Dockerfile
@@ -1,4 +1,4 @@
-FROM node:18-beta AS builder
+FROM node:18 AS builder
COPY /packages ./packages
COPY /package*.json ./
diff --git a/test/e2e/puppeteer-initial-cookies/actor/Dockerfile b/test/e2e/puppeteer-initial-cookies/actor/Dockerfile
index 505d02a8d..fba9ec484 100644
--- a/test/e2e/puppeteer-initial-cookies/actor/Dockerfile
+++ b/test/e2e/puppeteer-initial-cookies/actor/Dockerfile
@@ -1,4 +1,4 @@
-FROM node:18-beta AS builder
+FROM node:18 AS builder
COPY /packages ./packages
COPY /package*.json ./
diff --git a/test/e2e/puppeteer-page-info/actor/Dockerfile b/test/e2e/puppeteer-page-info/actor/Dockerfile
index 505d02a8d..fba9ec484 100644
--- a/test/e2e/puppeteer-page-info/actor/Dockerfile
+++ b/test/e2e/puppeteer-page-info/actor/Dockerfile
@@ -1,4 +1,4 @@
-FROM node:18-beta AS builder
+FROM node:18 AS builder
COPY /packages ./packages
COPY /package*.json ./
diff --git a/test/e2e/puppeteer-store-pagination-jquery/actor/Dockerfile b/test/e2e/puppeteer-store-pagination-jquery/actor/Dockerfile
index 505d02a8d..fba9ec484 100644
--- a/test/e2e/puppeteer-store-pagination-jquery/actor/Dockerfile
+++ b/test/e2e/puppeteer-store-pagination-jquery/actor/Dockerfile
@@ -1,4 +1,4 @@
-FROM node:18-beta AS builder
+FROM node:18 AS builder
COPY /packages ./packages
COPY /package*.json ./
diff --git a/test/e2e/puppeteer-store-pagination/actor/Dockerfile b/test/e2e/puppeteer-store-pagination/actor/Dockerfile
index 505d02a8d..fba9ec484 100644
--- a/test/e2e/puppeteer-store-pagination/actor/Dockerfile
+++ b/test/e2e/puppeteer-store-pagination/actor/Dockerfile
@@ -1,4 +1,4 @@
-FROM node:18-beta AS builder
+FROM node:18 AS builder
COPY /packages ./packages
COPY /package*.json ./
diff --git a/test/e2e/puppeteer-throw-on-ssl-errors/actor/Dockerfile b/test/e2e/puppeteer-throw-on-ssl-errors/actor/Dockerfile
index 505d02a8d..fba9ec484 100644
--- a/test/e2e/puppeteer-throw-on-ssl-errors/actor/Dockerfile
+++ b/test/e2e/puppeteer-throw-on-ssl-errors/actor/Dockerfile
@@ -1,4 +1,4 @@
-FROM node:18-beta AS builder
+FROM node:18 AS builder
COPY /packages ./packages
COPY /package*.json ./
diff --git a/test/e2e/session-rotation/actor/Dockerfile b/test/e2e/session-rotation/actor/Dockerfile
index 0be6b9d47..571491204 100644
--- a/test/e2e/session-rotation/actor/Dockerfile
+++ b/test/e2e/session-rotation/actor/Dockerfile
@@ -1,4 +1,4 @@
-FROM node:18-beta AS builder
+FROM node:18 AS builder
COPY /packages ./packages
COPY /package*.json ./