Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

fix(puppeteer): replace `page.waitForTimeout()` with `sleep()`

Martin Adámek52d7219acdc1
diff --git a/packages/puppeteer-crawler/src/internals/utils/puppeteer_utils.ts b/packages/puppeteer-crawler/src/internals/utils/puppeteer_utils.ts
index 154761391..2202fa764 100644
--- a/packages/puppeteer-crawler/src/internals/utils/puppeteer_utils.ts
+++ b/packages/puppeteer-crawler/src/internals/utils/puppeteer_utils.ts
@@ -26,7 +26,7 @@ import log_ from '@apify/log';
import type { Request } from '@crawlee/browser';
import { KeyValueStore, RequestState, validators, Configuration } from '@crawlee/browser';
import type { Dictionary, BatchAddRequestsResult } from '@crawlee/types';
-import type { CheerioRoot } from '@crawlee/utils';
+import { type CheerioRoot, sleep } from '@crawlee/utils';
import * as cheerio from 'cheerio';
import type { ProtocolMapping } from 'devtools-protocol/types/protocol-mapping.js';
import { getInjectableScript } from 'idcac-playwright';
@@ -524,7 +524,7 @@ export async function infiniteScroll(page: Page, options: InfiniteScrollOptions
let retry = 0;
while (!body && retry < 10) {
- await page.waitForTimeout(100);
+ await sleep(100);
body = await page.$('body');
retry++;
}
@@ -584,7 +584,7 @@ export async function infiniteScroll(page: Page, options: InfiniteScrollOptions
while (!finished) {
await doScroll();
- await page.waitForTimeout(250);
+ await sleep(250);
if (scrollDownAndUp) {
await page.mouse.wheel({ deltaY: -1000 });
}
diff --git a/test/e2e/puppeteer-default/actor/main.js b/test/e2e/puppeteer-default/actor/main.js
index 66ae746a9..2a4badbcb 100644
--- a/test/e2e/puppeteer-default/actor/main.js
+++ b/test/e2e/puppeteer-default/actor/main.js
@@ -11,7 +11,8 @@ await Actor.main(async () => {
preNavigationHooks: [(_ctx, goToOptions) => {
goToOptions.waitUntil = ['networkidle2'];
}],
- async requestHandler({ page, enqueueLinks, request }) {
+ async requestHandler({ page, enqueueLinks, request, infiniteScroll }) {
+ await infiniteScroll();
const { url } = request;
const pageTitle = await page.title();
await Dataset.pushData({ url, pageTitle });