Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Add gotoExtended back with better message

Ondra Urbanbfa4c84b2aaf
diff --git a/src/puppeteer_crawler.js b/src/puppeteer_crawler.js
index 700ce1af4..47560efa7 100644
--- a/src/puppeteer_crawler.js
+++ b/src/puppeteer_crawler.js
@@ -5,9 +5,10 @@ import BasicCrawler from './basic_crawler';
import PuppeteerPool from './puppeteer_pool';
import { addTimeoutToPromise } from './utils';
import { BASIC_CRAWLER_TIMEOUT_MULTIPLIER } from './constants';
+import { gotoExtended } from './puppeteer_utils';
const DEFAULT_OPTIONS = {
- gotoFunction: async ({ request, page }) => page.goto(request.url, { timeout: 60000 }),
+ gotoFunction: async ({ request, page }) => gotoExtended(page, request, { timeout: 60000 }),
handlePageTimeoutSecs: 60,
handleFailedRequestFunction: ({ request }) => {
const details = _.pick(request, 'id', 'url', 'method', 'uniqueKey');
diff --git a/src/puppeteer_utils.js b/src/puppeteer_utils.js
index 36a06dc55..2d652df86 100644
--- a/src/puppeteer_utils.js
+++ b/src/puppeteer_utils.js
@@ -404,12 +404,11 @@ export const gotoExtended = async (page, request, gotoOptions = {}) => {
checkParamPrototypeOrThrow(request, 'request', Request, 'Apify.Request');
checkParamOrThrow(gotoOptions, 'gotoOptions', 'Object');
- log.deprecated('Apify.utils.puppeteer.gotoExtended() has a high impact on performance '
- + 'in recent versions of Puppeteer. Use only when necessary.');
-
const { method, headers, payload } = request;
if (method !== 'GET' || payload || !_.isEmpty(headers)) {
+ log.deprecated('Using other request methods than GET, rewriting headers and adding payloads has a high impact on performance '
+ + 'in recent versions of Puppeteer. Use only when necessary.');
let wasCalled = false;
const interceptRequestHandler = async (interceptedRequest) => {
// We want to ensure that this won't get executed again in a case that there is a subsequent request