Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Add gotoExtended back with better message
Ondra Urbanbfa4c84b2aaf
diff --git a/src/puppeteer_crawler.js b/src/puppeteer_crawler.jsindex 700ce1af4..47560efa7 100644--- a/src/puppeteer_crawler.js+++ b/src/puppeteer_crawler.js@@ -5,9 +5,10 @@ import BasicCrawler from './basic_crawler';import PuppeteerPool from './puppeteer_pool';import { addTimeoutToPromise } from './utils';import { BASIC_CRAWLER_TIMEOUT_MULTIPLIER } from './constants';+import { gotoExtended } from './puppeteer_utils';const DEFAULT_OPTIONS = {- gotoFunction: async ({ request, page }) => page.goto(request.url, { timeout: 60000 }),+ gotoFunction: async ({ request, page }) => gotoExtended(page, request, { timeout: 60000 }),handlePageTimeoutSecs: 60,handleFailedRequestFunction: ({ request }) => {const details = _.pick(request, 'id', 'url', 'method', 'uniqueKey');diff --git a/src/puppeteer_utils.js b/src/puppeteer_utils.jsindex 36a06dc55..2d652df86 100644--- a/src/puppeteer_utils.js+++ b/src/puppeteer_utils.js@@ -404,12 +404,11 @@ export const gotoExtended = async (page, request, gotoOptions = {}) => {checkParamPrototypeOrThrow(request, 'request', Request, 'Apify.Request');checkParamOrThrow(gotoOptions, 'gotoOptions', 'Object');- log.deprecated('Apify.utils.puppeteer.gotoExtended() has a high impact on performance '- + 'in recent versions of Puppeteer. Use only when necessary.');-const { method, headers, payload } = request;if (method !== 'GET' || payload || !_.isEmpty(headers)) {+ log.deprecated('Using other request methods than GET, rewriting headers and adding payloads has a high impact on performance '+ + 'in recent versions of Puppeteer. Use only when necessary.');let wasCalled = false;const interceptRequestHandler = async (interceptedRequest) => {// We want to ensure that this won't get executed again in a case that there is a subsequent request