Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
fix: respect `<base>` when enqueuing (#1936)
Martin Adámekaeef57231c84
diff --git a/packages/browser-crawler/src/internals/browser-crawler.ts b/packages/browser-crawler/src/internals/browser-crawler.tsindex 8cd603f3f..665dbc05e 100644--- a/packages/browser-crawler/src/internals/browser-crawler.ts+++ b/packages/browser-crawler/src/internals/browser-crawler.ts@@ -1,29 +1,29 @@import { addTimeoutToPromise, tryCancel } from '@apify/timeout';import type {+ Awaitable,+ BasicCrawlerOptions,CrawlingContext,+ Dictionary,+ EnqueueLinksOptions,+ ErrorHandler,ProxyConfiguration,ProxyInfo,+ RequestHandler,RequestQueue,Session,- BasicCrawlerOptions,- Awaitable,- Dictionary,- RequestHandler,- ErrorHandler,- EnqueueLinksOptions,} from '@crawlee/basic';import {+ BASIC_CRAWLER_TIMEOUT_BUFFER_SECS,+ BasicCrawler,+ Configuration,cookieStringToToughCookie,enqueueLinks,EVENT_SESSION_RETIRED,handleRequestTimeout,- validators,- resolveBaseUrlForEnqueueLinksFiltering,- Configuration,- BASIC_CRAWLER_TIMEOUT_BUFFER_SECS,- BasicCrawler,- RequestState,tryAbsoluteURL,+ RequestState,+ resolveBaseUrlForEnqueueLinksFiltering,+ validators,} from '@crawlee/basic';import type {BrowserController,@@ -714,15 +714,21 @@ export async function browserCrawlerEnqueueLinks({* @ignore*/// eslint-disable-next-line @typescript-eslint/ban-types-async function extractUrlsFromPage(page: { $$eval: Function }, selector: string, baseUrl?: string): Promise<string[]> {+async function extractUrlsFromPage(page: { $$eval: Function }, selector: string, baseUrl: string): Promise<string[]> {const urls = await page.$$eval(selector, (linkEls: HTMLLinkElement[]) => linkEls.map((link) => link.getAttribute('href')).filter((href) => !!href)) ?? [];+ const [base] = await page.$$eval('base', (els: HTMLLinkElement[]) => els.map((el) => el.getAttribute('href')));+ const absoluteBaseUrl = base && tryAbsoluteURL(base, baseUrl);++ if (absoluteBaseUrl) {+ baseUrl = absoluteBaseUrl;+ }return urls.map((href: string) => {// Throw a meaningful error when only a relative URL would be extracted instead of waiting for the Request to fail later.const isHrefAbsolute = /^[a-z][a-z0-9+.-]*:/.test(href); // Grabbed this in 'is-absolute-url' package.if (!isHrefAbsolute && !baseUrl) {throw new Error(`An extracted URL: ${href} is relative and options.baseUrl is not set. `- + 'Use options.baseUrl in enqueueLinks() to automatically resolve relative URLs.');+ + 'Use options.baseUrl in enqueueLinks() to automatically resolve relative URLs.');}return baseUrldiff --git a/packages/cheerio-crawler/src/internals/cheerio-crawler.ts b/packages/cheerio-crawler/src/internals/cheerio-crawler.tsindex 472d6c207..9385b25f1 100644--- a/packages/cheerio-crawler/src/internals/cheerio-crawler.ts+++ b/packages/cheerio-crawler/src/internals/cheerio-crawler.ts@@ -234,7 +234,14 @@ export async function cheerioCrawlerEnqueueLinks({ options, $, requestQueue, ori* Extracts URLs from a given Cheerio object.* @ignore*/-function extractUrlsFromCheerio($: cheerio.CheerioAPI, selector: string, baseUrl?: string): string[] {+function extractUrlsFromCheerio($: cheerio.CheerioAPI, selector: string, baseUrl: string): string[] {+ const base = $('base').attr('href');+ const absoluteBaseUrl = base && tryAbsoluteURL(base, baseUrl);++ if (absoluteBaseUrl) {+ baseUrl = absoluteBaseUrl;+ }+return $(selector).map((_i, el) => $(el).attr('href')).get()diff --git a/test/e2e/cheerio-enqueue-links-base/actor/.actor/actor.json b/test/e2e/cheerio-enqueue-links-base/actor/.actor/actor.jsonnew file mode 100644index 000000000..0087602ef--- /dev/null+++ b/test/e2e/cheerio-enqueue-links-base/actor/.actor/actor.json@@ -0,0 +1,7 @@+{+ "actorSpecification": 1,+ "name": "test-cheerio-enqueue-links-base",+ "version": "0.0",+ "buildTag": "latest",+ "env": null+}diff --git a/test/e2e/cheerio-enqueue-links-base/actor/.gitignore b/test/e2e/cheerio-enqueue-links-base/actor/.gitignorenew file mode 100644index 000000000..ced7cbfc5--- /dev/null+++ b/test/e2e/cheerio-enqueue-links-base/actor/.gitignore@@ -0,0 +1,7 @@+.idea+.DS_Store+node_modules+package-lock.json+apify_storage+crawlee_storage+storagediff --git a/test/e2e/cheerio-enqueue-links-base/actor/Dockerfile b/test/e2e/cheerio-enqueue-links-base/actor/Dockerfilenew file mode 100644index 000000000..58806cec5--- /dev/null+++ b/test/e2e/cheerio-enqueue-links-base/actor/Dockerfile@@ -0,0 +1,16 @@+FROM apify/actor-node:18-beta++COPY packages ./packages+COPY package*.json ./++RUN npm --quiet set progress=false \+ && npm install --only=prod --no-optional \+ && npm update \+ && echo "Installed NPM packages:" \+ && (npm list --only=prod --no-optional --all || true) \+ && echo "Node.js version:" \+ && node --version \+ && echo "NPM version:" \+ && npm --version++COPY . ./diff --git a/test/e2e/cheerio-enqueue-links-base/actor/main.js b/test/e2e/cheerio-enqueue-links-base/actor/main.jsnew file mode 100644index 000000000..514fa7934--- /dev/null+++ b/test/e2e/cheerio-enqueue-links-base/actor/main.js@@ -0,0 +1,31 @@+import { Actor } from 'apify';+import { CheerioCrawler, Dataset } from '@crawlee/cheerio';++const mainOptions = {+ exit: Actor.isAtHome(),+ storage: process.env.STORAGE_IMPLEMENTATION === 'LOCAL' ? new (await import('@apify/storage-local')).ApifyStorageLocal() : undefined,+};++await Actor.main(async () => {+ const crawler = new CheerioCrawler({+ maxRequestsPerCrawl: 30,+ proxyConfiguration: await Actor.createProxyConfiguration(),+ async requestHandler({ $, enqueueLinks, request, log }) {+ const { url, loadedUrl } = request;++ const pageTitle = $('title').first().text();+ log.info(`URL: ${url}; LOADED_URL: ${loadedUrl}; TITLE: ${pageTitle}`);+ await Dataset.pushData({ url, loadedUrl, pageTitle });++ await enqueueLinks({+ globs: [+ 'https://www.jamesallen.com/about-us/**',+ 'https://www.jamesallen.com/terms-of-use/**',+ 'https://www.jamesallen.com/guarantee/**',+ ],+ });+ },+ });++ await crawler.run(['https://www.jamesallen.com/faq/']);+}, mainOptions);diff --git a/test/e2e/cheerio-enqueue-links-base/actor/package.json b/test/e2e/cheerio-enqueue-links-base/actor/package.jsonnew file mode 100644index 000000000..9c4711b45--- /dev/null+++ b/test/e2e/cheerio-enqueue-links-base/actor/package.json@@ -0,0 +1,29 @@+{+ "name": "test-cheerio-enqueue-links-base",+ "version": "0.0.1",+ "description": "Cheerio Crawler Test - Enqueue Links with <base href>",+ "dependencies": {+ "apify": "next",+ "@apify/storage-local": "^2.1.0",+ "@crawlee/basic": "file:./packages/basic-crawler",+ "@crawlee/browser-pool": "file:./packages/browser-pool",+ "@crawlee/http": "file:./packages/http-crawler",+ "@crawlee/cheerio": "file:./packages/cheerio-crawler",+ "@crawlee/core": "file:./packages/core",+ "@crawlee/memory-storage": "file:./packages/memory-storage",+ "@crawlee/types": "file:./packages/types",+ "@crawlee/utils": "file:./packages/utils",+ "deep-equal": "^2.0.5"+ },+ "overrides": {+ "apify": {+ "@crawlee/core": "file:./packages/core",+ "@crawlee/utils": "file:./packages/utils"+ }+ },+ "scripts": {+ "start": "node main.js"+ },+ "type": "module",+ "license": "ISC"+}diff --git a/test/e2e/cheerio-enqueue-links-base/test.mjs b/test/e2e/cheerio-enqueue-links-base/test.mjsnew file mode 100644index 000000000..502745fdd--- /dev/null+++ b/test/e2e/cheerio-enqueue-links-base/test.mjs@@ -0,0 +1,8 @@+import { initialize, getActorTestDir, runActor, expect } from '../tools.mjs';++const testActorDirname = getActorTestDir(import.meta.url);+await initialize(testActorDirname);++const { datasetItems } = await runActor(testActorDirname);++await expect(datasetItems.length === 14, `Enqueueing respects <base href>`);diff --git a/test/e2e/jsdom-react-ts/test.mjs b/test/e2e/jsdom-react-ts/test.mjsindex 93723ba2d..0b89623a5 100644--- a/test/e2e/jsdom-react-ts/test.mjs+++ b/test/e2e/jsdom-react-ts/test.mjs@@ -1,4 +1,6 @@-import { initialize, getActorTestDir, runActor, expect, validateDataset } from '../tools.mjs';+import { initialize, getActorTestDir, runActor, expect, validateDataset, skipTest } from '../tools.mjs';++await skipTest('target site no longer exists');const testActorDirname = getActorTestDir(import.meta.url);await initialize(testActorDirname);diff --git a/test/e2e/playwright-chromium-experimental-containers/actor/Dockerfile b/test/e2e/playwright-chromium-experimental-containers/actor/Dockerfileindex 0be6b9d47..571491204 100644--- a/test/e2e/playwright-chromium-experimental-containers/actor/Dockerfile+++ b/test/e2e/playwright-chromium-experimental-containers/actor/Dockerfile@@ -1,4 +1,4 @@-FROM node:18-beta AS builder+FROM node:18 AS builderCOPY /packages ./packagesCOPY /package*.json ./diff --git a/test/e2e/playwright-default/actor/Dockerfile b/test/e2e/playwright-default/actor/Dockerfileindex 0be6b9d47..571491204 100644--- a/test/e2e/playwright-default/actor/Dockerfile+++ b/test/e2e/playwright-default/actor/Dockerfile@@ -1,4 +1,4 @@-FROM node:18-beta AS builder+FROM node:18 AS builderCOPY /packages ./packagesCOPY /package*.json ./diff --git a/test/e2e/playwright-enqueue-links-base/actor/.actor/actor.json b/test/e2e/playwright-enqueue-links-base/actor/.actor/actor.jsonnew file mode 100644index 000000000..e40f74e8a--- /dev/null+++ b/test/e2e/playwright-enqueue-links-base/actor/.actor/actor.json@@ -0,0 +1,7 @@+{+ "actorSpecification": 1,+ "name": "test-playwright-enqueue-links-base",+ "version": "0.0",+ "buildTag": "latest",+ "env": null+}diff --git a/test/e2e/playwright-enqueue-links-base/actor/.gitignore b/test/e2e/playwright-enqueue-links-base/actor/.gitignorenew file mode 100644index 000000000..ced7cbfc5--- /dev/null+++ b/test/e2e/playwright-enqueue-links-base/actor/.gitignore@@ -0,0 +1,7 @@+.idea+.DS_Store+node_modules+package-lock.json+apify_storage+crawlee_storage+storagediff --git a/test/e2e/playwright-enqueue-links-base/actor/Dockerfile b/test/e2e/playwright-enqueue-links-base/actor/Dockerfilenew file mode 100644index 000000000..571491204--- /dev/null+++ b/test/e2e/playwright-enqueue-links-base/actor/Dockerfile@@ -0,0 +1,23 @@+FROM node:18 AS builder++COPY /packages ./packages+COPY /package*.json ./+RUN npm --quiet set progress=false \+ && npm install --only=prod --no-optional \+ && npm update++FROM apify/actor-node-playwright-chrome:18-beta++RUN rm -r node_modules+COPY --from=builder /node_modules ./node_modules+COPY --from=builder /packages ./packages+COPY --from=builder /package*.json ./+COPY /.actor ./.actor+COPY /main.js ./++RUN echo "Installed NPM packages:" \+ && (npm list --only=prod --no-optional --all || true) \+ && echo "Node.js version:" \+ && node --version \+ && echo "NPM version:" \+ && npm --versiondiff --git a/test/e2e/playwright-enqueue-links-base/actor/main.js b/test/e2e/playwright-enqueue-links-base/actor/main.jsnew file mode 100644index 000000000..9db543080--- /dev/null+++ b/test/e2e/playwright-enqueue-links-base/actor/main.js@@ -0,0 +1,32 @@+import { Actor } from 'apify';+import { PlaywrightCrawler, Dataset } from '@crawlee/playwright';++const mainOptions = {+ exit: Actor.isAtHome(),+ storage: process.env.STORAGE_IMPLEMENTATION === 'LOCAL' ? new (await import('@apify/storage-local')).ApifyStorageLocal() : undefined,+};++await Actor.main(async () => {+ const crawler = new PlaywrightCrawler({+ maxRequestsPerCrawl: 30,+ proxyConfiguration: await Actor.createProxyConfiguration(),+ async requestHandler({ parseWithCheerio, enqueueLinks, request, log }) {+ const { url, loadedUrl } = request;++ const $ = await parseWithCheerio();+ const pageTitle = $('title').first().text();+ log.info(`URL: ${url}; LOADED_URL: ${loadedUrl}; TITLE: ${pageTitle}`);+ await Dataset.pushData({ url, loadedUrl, pageTitle });++ await enqueueLinks({+ globs: [+ 'https://www.jamesallen.com/about-us/**',+ 'https://www.jamesallen.com/terms-of-use/**',+ 'https://www.jamesallen.com/guarantee/**',+ ],+ });+ },+ });++ await crawler.run(['https://www.jamesallen.com/faq/']);+}, mainOptions);diff --git a/test/e2e/playwright-enqueue-links-base/actor/package.json b/test/e2e/playwright-enqueue-links-base/actor/package.jsonnew file mode 100644index 000000000..bae23adab--- /dev/null+++ b/test/e2e/playwright-enqueue-links-base/actor/package.json@@ -0,0 +1,30 @@+{+ "name": "test-playwright-enqueue-links-base",+ "version": "0.0.1",+ "description": "Playwright Test - Enqueue Links with <base href>",+ "dependencies": {+ "apify": "next",+ "@apify/storage-local": "^2.1.0",+ "@crawlee/basic": "file:./packages/basic-crawler",+ "@crawlee/browser": "file:./packages/browser-crawler",+ "@crawlee/browser-pool": "file:./packages/browser-pool",+ "@crawlee/core": "file:./packages/core",+ "@crawlee/memory-storage": "file:./packages/memory-storage",+ "@crawlee/playwright": "file:./packages/playwright-crawler",+ "@crawlee/types": "file:./packages/types",+ "@crawlee/utils": "file:./packages/utils",+ "deep-equal": "^2.0.5",+ "playwright": "*"+ },+ "overrides": {+ "apify": {+ "@crawlee/core": "file:./packages/core",+ "@crawlee/utils": "file:./packages/utils"+ }+ },+ "scripts": {+ "start": "node main.js"+ },+ "type": "module",+ "license": "ISC"+}diff --git a/test/e2e/playwright-enqueue-links-base/test.mjs b/test/e2e/playwright-enqueue-links-base/test.mjsnew file mode 100644index 000000000..502745fdd--- /dev/null+++ b/test/e2e/playwright-enqueue-links-base/test.mjs@@ -0,0 +1,8 @@+import { initialize, getActorTestDir, runActor, expect } from '../tools.mjs';++const testActorDirname = getActorTestDir(import.meta.url);+await initialize(testActorDirname);++const { datasetItems } = await runActor(testActorDirname);++await expect(datasetItems.length === 14, `Enqueueing respects <base href>`);diff --git a/test/e2e/playwright-enqueue-links/actor/Dockerfile b/test/e2e/playwright-enqueue-links/actor/Dockerfileindex 0be6b9d47..571491204 100644--- a/test/e2e/playwright-enqueue-links/actor/Dockerfile+++ b/test/e2e/playwright-enqueue-links/actor/Dockerfile@@ -1,4 +1,4 @@-FROM node:18-beta AS builder+FROM node:18 AS builderCOPY /packages ./packagesCOPY /package*.json ./diff --git a/test/e2e/playwright-firefox-experimental-containers/actor/Dockerfile b/test/e2e/playwright-firefox-experimental-containers/actor/Dockerfileindex dd42e7f17..47d04fac5 100644--- a/test/e2e/playwright-firefox-experimental-containers/actor/Dockerfile+++ b/test/e2e/playwright-firefox-experimental-containers/actor/Dockerfile@@ -1,4 +1,4 @@-FROM node:18-beta AS builder+FROM node:18 AS builderCOPY /packages ./packagesCOPY /package*.json ./diff --git a/test/e2e/playwright-initial-cookies/actor/Dockerfile b/test/e2e/playwright-initial-cookies/actor/Dockerfileindex 0be6b9d47..571491204 100644--- a/test/e2e/playwright-initial-cookies/actor/Dockerfile+++ b/test/e2e/playwright-initial-cookies/actor/Dockerfile@@ -1,4 +1,4 @@-FROM node:18-beta AS builder+FROM node:18 AS builderCOPY /packages ./packagesCOPY /package*.json ./diff --git a/test/e2e/playwright-multi-run/actor/Dockerfile b/test/e2e/playwright-multi-run/actor/Dockerfileindex 0be6b9d47..571491204 100644--- a/test/e2e/playwright-multi-run/actor/Dockerfile+++ b/test/e2e/playwright-multi-run/actor/Dockerfile@@ -1,4 +1,4 @@-FROM node:18-beta AS builder+FROM node:18 AS builderCOPY /packages ./packagesCOPY /package*.json ./diff --git a/test/e2e/proxy-rotation/actor/Dockerfile b/test/e2e/proxy-rotation/actor/Dockerfileindex 505d02a8d..fba9ec484 100644--- a/test/e2e/proxy-rotation/actor/Dockerfile+++ b/test/e2e/proxy-rotation/actor/Dockerfile@@ -1,4 +1,4 @@-FROM node:18-beta AS builder+FROM node:18 AS builderCOPY /packages ./packagesCOPY /package*.json ./diff --git a/test/e2e/puppeteer-default/actor/Dockerfile b/test/e2e/puppeteer-default/actor/Dockerfileindex 505d02a8d..fba9ec484 100644--- a/test/e2e/puppeteer-default/actor/Dockerfile+++ b/test/e2e/puppeteer-default/actor/Dockerfile@@ -1,4 +1,4 @@-FROM node:18-beta AS builder+FROM node:18 AS builderCOPY /packages ./packagesCOPY /package*.json ./diff --git a/test/e2e/puppeteer-enqueue-links/actor/Dockerfile b/test/e2e/puppeteer-enqueue-links/actor/Dockerfileindex 505d02a8d..fba9ec484 100644--- a/test/e2e/puppeteer-enqueue-links/actor/Dockerfile+++ b/test/e2e/puppeteer-enqueue-links/actor/Dockerfile@@ -1,4 +1,4 @@-FROM node:18-beta AS builder+FROM node:18 AS builderCOPY /packages ./packagesCOPY /package*.json ./diff --git a/test/e2e/puppeteer-ignore-ssl-errors/actor/Dockerfile b/test/e2e/puppeteer-ignore-ssl-errors/actor/Dockerfileindex 505d02a8d..fba9ec484 100644--- a/test/e2e/puppeteer-ignore-ssl-errors/actor/Dockerfile+++ b/test/e2e/puppeteer-ignore-ssl-errors/actor/Dockerfile@@ -1,4 +1,4 @@-FROM node:18-beta AS builder+FROM node:18 AS builderCOPY /packages ./packagesCOPY /package*.json ./diff --git a/test/e2e/puppeteer-initial-cookies/actor/Dockerfile b/test/e2e/puppeteer-initial-cookies/actor/Dockerfileindex 505d02a8d..fba9ec484 100644--- a/test/e2e/puppeteer-initial-cookies/actor/Dockerfile+++ b/test/e2e/puppeteer-initial-cookies/actor/Dockerfile@@ -1,4 +1,4 @@-FROM node:18-beta AS builder+FROM node:18 AS builderCOPY /packages ./packagesCOPY /package*.json ./diff --git a/test/e2e/puppeteer-page-info/actor/Dockerfile b/test/e2e/puppeteer-page-info/actor/Dockerfileindex 505d02a8d..fba9ec484 100644--- a/test/e2e/puppeteer-page-info/actor/Dockerfile+++ b/test/e2e/puppeteer-page-info/actor/Dockerfile@@ -1,4 +1,4 @@-FROM node:18-beta AS builder+FROM node:18 AS builderCOPY /packages ./packagesCOPY /package*.json ./diff --git a/test/e2e/puppeteer-store-pagination-jquery/actor/Dockerfile b/test/e2e/puppeteer-store-pagination-jquery/actor/Dockerfileindex 505d02a8d..fba9ec484 100644--- a/test/e2e/puppeteer-store-pagination-jquery/actor/Dockerfile+++ b/test/e2e/puppeteer-store-pagination-jquery/actor/Dockerfile@@ -1,4 +1,4 @@-FROM node:18-beta AS builder+FROM node:18 AS builderCOPY /packages ./packagesCOPY /package*.json ./diff --git a/test/e2e/puppeteer-store-pagination/actor/Dockerfile b/test/e2e/puppeteer-store-pagination/actor/Dockerfileindex 505d02a8d..fba9ec484 100644--- a/test/e2e/puppeteer-store-pagination/actor/Dockerfile+++ b/test/e2e/puppeteer-store-pagination/actor/Dockerfile@@ -1,4 +1,4 @@-FROM node:18-beta AS builder+FROM node:18 AS builderCOPY /packages ./packagesCOPY /package*.json ./diff --git a/test/e2e/puppeteer-throw-on-ssl-errors/actor/Dockerfile b/test/e2e/puppeteer-throw-on-ssl-errors/actor/Dockerfileindex 505d02a8d..fba9ec484 100644--- a/test/e2e/puppeteer-throw-on-ssl-errors/actor/Dockerfile+++ b/test/e2e/puppeteer-throw-on-ssl-errors/actor/Dockerfile@@ -1,4 +1,4 @@-FROM node:18-beta AS builder+FROM node:18 AS builderCOPY /packages ./packagesCOPY /package*.json ./diff --git a/test/e2e/session-rotation/actor/Dockerfile b/test/e2e/session-rotation/actor/Dockerfileindex 0be6b9d47..571491204 100644--- a/test/e2e/session-rotation/actor/Dockerfile+++ b/test/e2e/session-rotation/actor/Dockerfile@@ -1,4 +1,4 @@-FROM node:18-beta AS builder+FROM node:18 AS builderCOPY /packages ./packagesCOPY /package*.json ./