Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
fix(puppeteer): replace `page.waitForTimeout()` with `sleep()`
Martin Adámek52d7219acdc1
diff --git a/packages/puppeteer-crawler/src/internals/utils/puppeteer_utils.ts b/packages/puppeteer-crawler/src/internals/utils/puppeteer_utils.tsindex 154761391..2202fa764 100644--- a/packages/puppeteer-crawler/src/internals/utils/puppeteer_utils.ts+++ b/packages/puppeteer-crawler/src/internals/utils/puppeteer_utils.ts@@ -26,7 +26,7 @@ import log_ from '@apify/log';import type { Request } from '@crawlee/browser';import { KeyValueStore, RequestState, validators, Configuration } from '@crawlee/browser';import type { Dictionary, BatchAddRequestsResult } from '@crawlee/types';-import type { CheerioRoot } from '@crawlee/utils';+import { type CheerioRoot, sleep } from '@crawlee/utils';import * as cheerio from 'cheerio';import type { ProtocolMapping } from 'devtools-protocol/types/protocol-mapping.js';import { getInjectableScript } from 'idcac-playwright';@@ -524,7 +524,7 @@ export async function infiniteScroll(page: Page, options: InfiniteScrollOptionslet retry = 0;while (!body && retry < 10) {- await page.waitForTimeout(100);+ await sleep(100);body = await page.$('body');retry++;}@@ -584,7 +584,7 @@ export async function infiniteScroll(page: Page, options: InfiniteScrollOptionswhile (!finished) {await doScroll();- await page.waitForTimeout(250);+ await sleep(250);if (scrollDownAndUp) {await page.mouse.wheel({ deltaY: -1000 });}diff --git a/test/e2e/puppeteer-default/actor/main.js b/test/e2e/puppeteer-default/actor/main.jsindex 66ae746a9..2a4badbcb 100644--- a/test/e2e/puppeteer-default/actor/main.js+++ b/test/e2e/puppeteer-default/actor/main.js@@ -11,7 +11,8 @@ await Actor.main(async () => {preNavigationHooks: [(_ctx, goToOptions) => {goToOptions.waitUntil = ['networkidle2'];}],- async requestHandler({ page, enqueueLinks, request }) {+ async requestHandler({ page, enqueueLinks, request, infiniteScroll }) {+ await infiniteScroll();const { url } = request;const pageTitle = await page.title();await Dataset.pushData({ url, pageTitle });