Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

feat: add `waitForSelector` context helper + `parseWithCheerio` in adaptive crawler (#2522)

Martin Adámek6f88e738d43a
diff --git a/biome.json b/biome.json
index a4c3b3e64..7862480ba 100644
--- a/biome.json
+++ b/biome.json
@@ -13,7 +13,7 @@
"formatter": {
"quoteStyle": "single",
"semicolons": "always",
- "trailingComma": "all",
+ "trailingCommas": "all",
"lineWidth": 120,
"indentStyle": "space",
"indentWidth": 4,
diff --git a/packages/cheerio-crawler/src/internals/cheerio-crawler.ts b/packages/cheerio-crawler/src/internals/cheerio-crawler.ts
index 157fb9c67..49009ff86 100644
--- a/packages/cheerio-crawler/src/internals/cheerio-crawler.ts
+++ b/packages/cheerio-crawler/src/internals/cheerio-crawler.ts
@@ -14,7 +14,7 @@ import type {
} from '@crawlee/http';
import { HttpCrawler, enqueueLinks, Router, resolveBaseUrlForEnqueueLinksFiltering } from '@crawlee/http';
import type { Dictionary } from '@crawlee/types';
-import { extractUrlsFromCheerio } from '@crawlee/utils';
+import { type CheerioRoot, extractUrlsFromCheerio } from '@crawlee/utils';
import type { CheerioOptions } from 'cheerio';
import * as cheerio from 'cheerio';
import { DomHandler } from 'htmlparser2';
@@ -45,20 +45,35 @@ export interface CheerioCrawlingContext<
*/
$: cheerio.CheerioAPI;
+ /**
+ * Wait for an element matching the selector to appear. Timeout is ignored.
+ *
+ * **Example usage:**
+ * ```ts
+ * async requestHandler({ waitForSelector, parseWithCheerio }) {
+ * await waitForSelector('article h1');
+ * const $ = await parseWithCheerio();
+ * const title = $('title').text();
+ * });
+ * ```
+ */
+ waitForSelector(selector: string, timeoutMs?: number): Promise<void>;
+
/**
* Returns Cheerio handle, this is here to unify the crawler API, so they all have this handy method.
* It has the same return type as the `$` context property, use it only if you are abstracting your workflow to
* support different context types in one handler.
+ * When provided with the `selector` argument, it will throw if it's not available.
*
* **Example usage:**
- * ```javascript
+ * ```ts
* async requestHandler({ parseWithCheerio }) {
* const $ = await parseWithCheerio();
* const title = $('title').text();
* });
* ```
*/
- parseWithCheerio(): Promise<cheerio.CheerioAPI>;
+ parseWithCheerio(selector?: string, timeoutMs?: number): Promise<CheerioRoot>;
}
export type CheerioRequestHandler<
@@ -204,7 +219,19 @@ export class CheerioCrawler extends HttpCrawler<CheerioCrawlingContext> {
}
protected override async _runRequestHandler(context: CheerioCrawlingContext) {
- context.parseWithCheerio = async () => Promise.resolve(context.$);
+ context.waitForSelector = async (selector?: string, _timeoutMs?: number) => {
+ if (context.$(selector).get().length === 0) {
+ throw new Error(`Selector '${selector}' not found.`);
+ }
+ };
+ context.parseWithCheerio = async (selector?: string, timeoutMs?: number) => {
+ if (selector) {
+ await context.waitForSelector(selector, timeoutMs);
+ }
+
+ return context.$;
+ };
+
await super._runRequestHandler(context);
}
}
diff --git a/packages/core/src/crawlers/crawler_commons.ts b/packages/core/src/crawlers/crawler_commons.ts
index 37d54e7b2..e32bb329b 100644
--- a/packages/core/src/crawlers/crawler_commons.ts
+++ b/packages/core/src/crawlers/crawler_commons.ts
@@ -15,6 +15,15 @@ import { KeyValueStore } from '../storages';
export interface RestrictedCrawlingContext<UserData extends Dictionary = Dictionary>
// we need `Record<string & {}, unknown>` here, otherwise `Omit<Context>` is resolved badly
extends Record<string & {}, unknown> {
+ id: string;
+ session?: Session;
+
+ /**
+ * An object with information about currently used proxy by the crawler
+ * and configured by the {@apilink ProxyConfiguration} class.
+ */
+ proxyInfo?: ProxyInfo;
+
/**
* The original {@apilink Request} object.
*/
@@ -86,15 +95,6 @@ export interface RestrictedCrawlingContext<UserData extends Dictionary = Diction
export interface CrawlingContext<Crawler = unknown, UserData extends Dictionary = Dictionary>
extends RestrictedCrawlingContext<UserData> {
- id: string;
- session?: Session;
-
- /**
- * An object with information about currently used proxy by the crawler
- * and configured by the {@apilink ProxyConfiguration} class.
- */
- proxyInfo?: ProxyInfo;
-
crawler: Crawler;
/**
diff --git a/packages/http-crawler/src/internals/http-crawler.ts b/packages/http-crawler/src/internals/http-crawler.ts
index 0c9b166d1..0f7ca2d1a 100644
--- a/packages/http-crawler/src/internals/http-crawler.ts
+++ b/packages/http-crawler/src/internals/http-crawler.ts
@@ -28,7 +28,7 @@ import {
SessionError,
} from '@crawlee/basic';
import type { Awaitable, Dictionary } from '@crawlee/types';
-import { RETRY_CSS_SELECTORS, gotScraping } from '@crawlee/utils';
+import { RETRY_CSS_SELECTORS, gotScraping, type CheerioRoot } from '@crawlee/utils';
import * as cheerio from 'cheerio';
import type { RequestLike, ResponseLike } from 'content-type';
import contentTypeParser from 'content-type';
@@ -215,7 +215,33 @@ export interface InternalHttpCrawlingContext<
contentType: { type: string; encoding: BufferEncoding };
response: PlainResponse;
- parseWithCheerio(): Promise<cheerio.CheerioAPI>;
+ /**
+ * Wait for an element matching the selector to appear. Timeout is ignored.
+ *
+ * **Example usage:**
+ * ```ts
+ * async requestHandler({ waitForSelector, parseWithCheerio }) {
+ * await waitForSelector('article h1');
+ * const $ = await parseWithCheerio();
+ * const title = $('title').text();
+ * });
+ * ```
+ */
+ waitForSelector(selector: string, timeoutMs?: number): Promise<void>;
+
+ /**
+ * Returns Cheerio handle for `page.content()`, allowing to work with the data same way as with {@apilink CheerioCrawler}.
+ * When provided with the `selector` argument, it will throw if it's not available.
+ *
+ * **Example usage:**
+ * ```ts
+ * async requestHandler({ parseWithCheerio }) {
+ * const $ = await parseWithCheerio();
+ * const title = $('title').text();
+ * });
+ * ```
+ */
+ parseWithCheerio(selector?: string, timeoutMs?: number): Promise<CheerioRoot>;
}
export interface HttpCrawlingContext<UserData extends Dictionary = any, JSONData extends JsonValue = any>
@@ -488,7 +514,22 @@ export class HttpCrawler<
tryCancel();
// `??=` because descendant classes may already set optimized version
- crawlingContext.parseWithCheerio ??= async () => cheerio.load(parsed.body!.toString());
+ crawlingContext.waitForSelector ??= async (selector?: string, _timeoutMs?: number) => {
+ const $ = cheerio.load(parsed.body!.toString());
+
+ if ($(selector).get().length === 0) {
+ throw new Error(`Selector '${selector}' not found.`);
+ }
+ };
+ crawlingContext.parseWithCheerio ??= async (selector?: string, timeoutMs?: number) => {
+ const $ = cheerio.load(parsed.body!.toString());
+
+ if (selector) {
+ await crawlingContext.waitForSelector(selector, timeoutMs);
+ }
+
+ return $;
+ };
if (this.useSessionPool) {
this._throwOnBlockedRequest(crawlingContext.session!, response.statusCode!);
diff --git a/packages/jsdom-crawler/src/internals/jsdom-crawler.ts b/packages/jsdom-crawler/src/internals/jsdom-crawler.ts
index 3e97c4718..46d7425ce 100644
--- a/packages/jsdom-crawler/src/internals/jsdom-crawler.ts
+++ b/packages/jsdom-crawler/src/internals/jsdom-crawler.ts
@@ -22,6 +22,7 @@ import {
tryAbsoluteURL,
} from '@crawlee/http';
import type { Dictionary } from '@crawlee/types';
+import { type CheerioRoot, sleep } from '@crawlee/utils';
import * as cheerio from 'cheerio';
import type { DOMWindow } from 'jsdom';
import { JSDOM, ResourceLoader, VirtualConsole } from 'jsdom';
@@ -58,8 +59,24 @@ export interface JSDOMCrawlingContext<
window: DOMWindow;
document: Document;
+ /**
+ * Wait for an element matching the selector to appear. Timeout is ignored.
+ * Timeout defaults to 5s.
+ *
+ * **Example usage:**
+ * ```ts
+ * async requestHandler({ waitForSelector, parseWithCheerio }) {
+ * await waitForSelector('article h1');
+ * const $ = await parseWithCheerio();
+ * const title = $('title').text();
+ * });
+ * ```
+ */
+ waitForSelector(selector: string, timeoutMs?: number): Promise<void>;
+
/**
* Returns Cheerio handle, allowing to work with the data same way as with {@apilink CheerioCrawler}.
+ * When provided with the `selector` argument, it will first look for the selector with a 5s timeout.
*
* **Example usage:**
* ```javascript
@@ -69,7 +86,7 @@ export interface JSDOMCrawlingContext<
* });
* ```
*/
- parseWithCheerio(): Promise<cheerio.CheerioAPI>;
+ parseWithCheerio(selector?: string, timeoutMs?: number): Promise<CheerioRoot>;
}
export type JSDOMRequestHandler<
@@ -294,7 +311,28 @@ export class JSDOMCrawler extends HttpCrawler<JSDOMCrawlingContext> {
}
override async _runRequestHandler(context: JSDOMCrawlingContext) {
- context.parseWithCheerio = async () => Promise.resolve(cheerio.load(context.body));
+ context.waitForSelector = async (selector: string, timeoutMs = 5_000) => {
+ const $ = cheerio.load(context.body);
+
+ if ($(selector).get().length === 0) {
+ if (timeoutMs) {
+ await sleep(50);
+ return context.waitForSelector(selector, Math.max(timeoutMs - 50, 0));
+ }
+
+ throw new Error(`Selector '${selector}' not found.`);
+ }
+ };
+ context.parseWithCheerio = async (selector?: string, _timeoutMs = 5_000) => {
+ const $ = cheerio.load(context.body);
+
+ if (selector && $(selector).get().length === 0) {
+ throw new Error(`Selector '${selector}' not found.`);
+ }
+
+ return $;
+ };
+
await super._runRequestHandler(context);
}
}
diff --git a/packages/linkedom-crawler/src/internals/linkedom-crawler.ts b/packages/linkedom-crawler/src/internals/linkedom-crawler.ts
index 8833f2595..ddb73bcad 100644
--- a/packages/linkedom-crawler/src/internals/linkedom-crawler.ts
+++ b/packages/linkedom-crawler/src/internals/linkedom-crawler.ts
@@ -20,7 +20,8 @@ import {
tryAbsoluteURL,
} from '@crawlee/http';
import type { Dictionary } from '@crawlee/types';
-import type * as cheerio from 'cheerio';
+import { type CheerioRoot, sleep } from '@crawlee/utils';
+import * as cheerio from 'cheerio';
// @ts-expect-error This throws a compilation error due to TypeScript not inferring the module has CJS versions too
import { DOMParser } from 'linkedom/cached';
@@ -53,8 +54,24 @@ export interface LinkeDOMCrawlingContext<
// even though it's not technically 100% correct
document: Document;
+ /**
+ * Wait for an element matching the selector to appear. Timeout is ignored.
+ * Timeout defaults to 5s.
+ *
+ * **Example usage:**
+ * ```ts
+ * async requestHandler({ waitForSelector, parseWithCheerio }) {
+ * await waitForSelector('article h1');
+ * const $ = await parseWithCheerio();
+ * const title = $('title').text();
+ * });
+ * ```
+ */
+ waitForSelector(selector: string, timeoutMs?: number): Promise<void>;
+
/**
* Returns Cheerio handle, allowing to work with the data same way as with {@apilink CheerioCrawler}.
+ * When provided with the `selector` argument, it will first look for the selector with a 5s timeout.
*
* **Example usage:**
* ```javascript
@@ -64,7 +81,7 @@ export interface LinkeDOMCrawlingContext<
* });
* ```
*/
- parseWithCheerio(): Promise<cheerio.CheerioAPI>;
+ parseWithCheerio(selector?: string, timeoutMs?: number): Promise<CheerioRoot>;
}
export type LinkeDOMRequestHandler<
@@ -175,6 +192,32 @@ export class LinkeDOMCrawler extends HttpCrawler<LinkeDOMCrawlingContext> {
},
};
}
+
+ override async _runRequestHandler(context: LinkeDOMCrawlingContext) {
+ context.waitForSelector = async (selector: string, timeoutMs = 5_000) => {
+ const $ = cheerio.load(context.body);
+
+ if ($(selector).get().length === 0) {
+ if (timeoutMs) {
+ await sleep(50);
+ return context.waitForSelector(selector, Math.max(timeoutMs - 50, 0));
+ }
+
+ throw new Error(`Selector '${selector}' not found.`);
+ }
+ };
+ context.parseWithCheerio = async (selector?: string, _timeoutMs = 5_000) => {
+ const $ = cheerio.load(context.body);
+
+ if (selector && $(selector).get().length === 0) {
+ throw new Error(`Selector '${selector}' not found.`);
+ }
+
+ return $;
+ };
+
+ await super._runRequestHandler(context);
+ }
}
interface EnqueueLinksInternalOptions {
diff --git a/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.ts b/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.ts
index d6e0f8778..9ce99a8b9 100644
--- a/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.ts
+++ b/packages/playwright-crawler/src/internals/adaptive-playwright-crawler.ts
@@ -10,7 +10,7 @@ import type {
} from '@crawlee/core';
import { Configuration, RequestHandlerResult, Router, Statistics, withCheckedStorageAccess } from '@crawlee/core';
import type { Awaitable, Dictionary } from '@crawlee/types';
-import { extractUrlsFromCheerio } from '@crawlee/utils';
+import { type CheerioRoot, extractUrlsFromCheerio } from '@crawlee/utils';
import { load, type Cheerio, type Element } from 'cheerio';
import isEqual from 'lodash.isequal';
@@ -81,8 +81,38 @@ class AdaptivePlaywrightCrawlerStatistics extends Statistics {
export interface AdaptivePlaywrightCrawlerContext extends RestrictedCrawlingContext {
/**
* Wait for an element matching the selector to appear and return a Cheerio object of matched elements.
+ * Timeout defaults to 5s.
*/
- querySelector: (selector: string, timeoutMs?: number) => Awaitable<Cheerio<Element>>;
+ querySelector(selector: string, timeoutMs?: number): Promise<Cheerio<Element>>;
+
+ /**
+ * Wait for an element matching the selector to appear.
+ * Timeout defaults to 5s.
+ *
+ * **Example usage:**
+ * ```ts
+ * async requestHandler({ waitForSelector, parseWithCheerio }) {
+ * await waitForSelector('article h1');
+ * const $ = await parseWithCheerio();
+ * const title = $('title').text();
+ * });
+ * ```
+ */
+ waitForSelector(selector: string, timeoutMs?: number): Promise<void>;
+
+ /**
+ * Returns Cheerio handle for `page.content()`, allowing to work with the data same way as with {@apilink CheerioCrawler}.
+ * When provided with the `selector` argument, it will first look for the selector with a 5s timeout.
+ *
+ * **Example usage:**
+ * ```ts
+ * async requestHandler({ parseWithCheerio }) {
+ * const $ = await parseWithCheerio();
+ * const title = $('title').text();
+ * });
+ * ```
+ */
+ parseWithCheerio(selector?: string, timeoutMs?: number): Promise<CheerioRoot>;
}
export interface AdaptivePlaywrightCrawlerOptions
@@ -102,8 +132,9 @@ export interface AdaptivePlaywrightCrawlerOptions
/**
* Specifies the frequency of rendering type detection checks - 0.1 means roughly 10% of requests.
+ * Defaults to 0.1 (so 10%).
*/
- renderingTypeDetectionRatio: number;
+ renderingTypeDetectionRatio?: number;
/**
* An optional callback that is called on dataset items found by the request handler in plain HTTP mode.
@@ -171,18 +202,20 @@ export class AdaptivePlaywrightCrawler extends PlaywrightCrawler {
Router.create<AdaptivePlaywrightCrawlerContext>();
constructor(
- {
+ options: AdaptivePlaywrightCrawlerOptions,
+ override readonly config = Configuration.getGlobalConfig(),
+ ) {
+ const {
requestHandler,
- renderingTypeDetectionRatio,
+ renderingTypeDetectionRatio = 0.1,
renderingTypePredictor,
resultChecker,
resultComparator,
statisticsOptions,
- ...options
- }: AdaptivePlaywrightCrawlerOptions,
- override readonly config = Configuration.getGlobalConfig(),
- ) {
- super(options, config);
+ ...rest
+ } = options;
+
+ super(rest, config);
this.adaptiveRequestHandler = requestHandler ?? this.router;
this.renderingTypePredictor =
renderingTypePredictor ?? new RenderingTypePredictor({ detectionRatio: renderingTypeDetectionRatio });
@@ -218,19 +251,19 @@ export class AdaptivePlaywrightCrawler extends PlaywrightCrawler {
const shouldDetectRenderingType = Math.random() < renderingTypePrediction.detectionProbabilityRecommendation;
if (!shouldDetectRenderingType) {
- crawlingContext.log.info(
+ crawlingContext.log.debug(
`Predicted rendering type ${renderingTypePrediction.renderingType} for ${crawlingContext.request.url}`,
);
}
if (renderingTypePrediction.renderingType === 'static' && !shouldDetectRenderingType) {
- crawlingContext.log.info(`Running HTTP-only request handler for ${crawlingContext.request.url}`);
+ crawlingContext.log.debug(`Running HTTP-only request handler for ${crawlingContext.request.url}`);
this.stats.trackHttpOnlyRequestHandlerRun();
const plainHTTPRun = await this.runRequestHandlerWithPlainHTTP(crawlingContext);
if (plainHTTPRun.ok && this.resultChecker(plainHTTPRun.result)) {
- crawlingContext.log.info(`HTTP-only request handler succeeded for ${crawlingContext.request.url}`);
+ crawlingContext.log.debug(`HTTP-only request handler succeeded for ${crawlingContext.request.url}`);
await this.commitResult(crawlingContext, plainHTTPRun.result);
return;
}
@@ -247,7 +280,7 @@ export class AdaptivePlaywrightCrawler extends PlaywrightCrawler {
}
}
- crawlingContext.log.info(`Running browser request handler for ${crawlingContext.request.url}`);
+ crawlingContext.log.debug(`Running browser request handler for ${crawlingContext.request.url}`);
this.stats.trackBrowserRequestHandlerRun();
const browserRun = await this.runRequestHandlerInBrowser(crawlingContext);
@@ -258,7 +291,7 @@ export class AdaptivePlaywrightCrawler extends PlaywrightCrawler {
await this.commitResult(crawlingContext, browserRun.result);
if (shouldDetectRenderingType) {
- crawlingContext.log.info(`Detecting rendering type for ${crawlingContext.request.url}`);
+ crawlingContext.log.debug(`Detecting rendering type for ${crawlingContext.request.url}`);
const plainHTTPRun = await this.runRequestHandlerWithPlainHTTP(crawlingContext);
const detectionResult: RenderingType = (() => {
@@ -273,7 +306,7 @@ export class AdaptivePlaywrightCrawler extends PlaywrightCrawler {
return 'clientOnly';
})();
- crawlingContext.log.info(`Detected rendering type ${detectionResult} for ${crawlingContext.request.url}`);
+ crawlingContext.log.debug(`Detected rendering type ${detectionResult} for ${crawlingContext.request.url}`);
this.renderingTypePredictor.storeResult(url, crawlingContext.request.label, detectionResult);
}
}
@@ -326,16 +359,34 @@ export class AdaptivePlaywrightCrawler extends PlaywrightCrawler {
},
() =>
this.adaptiveRequestHandler({
+ id: crawlingContext.id,
+ session: crawlingContext.session,
+ proxyInfo: crawlingContext.proxyInfo,
request: crawlingContext.request,
log: crawlingContext.log,
- querySelector: async (selector, timeoutMs) => {
+ querySelector: async (selector, timeoutMs = 5_000) => {
+ const locator = playwrightContext.page.locator(selector).first();
+ await locator.waitFor({ timeout: timeoutMs, state: 'attached' });
+ const $ = await playwrightContext.parseWithCheerio();
+
+ return $(selector) as Cheerio<Element>;
+ },
+ async waitForSelector(selector, timeoutMs = 5_000) {
const locator = playwrightContext.page.locator(selector).first();
await locator.waitFor({ timeout: timeoutMs, state: 'attached' });
- return (await playwrightContext.parseWithCheerio())(
- selector,
- ) as Cheerio<Element>;
},
- enqueueLinks: async (options = {}) => {
+ async parseWithCheerio(
+ selector?: string,
+ timeoutMs = 5_000,
+ ): Promise<CheerioRoot> {
+ if (selector) {
+ const locator = playwrightContext.page.locator(selector).first();
+ await locator.waitFor({ timeout: timeoutMs, state: 'attached' });
+ }
+
+ return playwrightContext.parseWithCheerio();
+ },
+ async enqueueLinks(options = {}) {
const selector = options.selector ?? 'a';
const locator = playwrightContext.page.locator(selector).first();
await locator.waitFor();
@@ -388,12 +439,29 @@ export class AdaptivePlaywrightCrawler extends PlaywrightCrawler {
addTimeoutToPromise(
async () =>
this.adaptiveRequestHandler({
+ id: crawlingContext.id,
+ session: crawlingContext.session,
+ proxyInfo: crawlingContext.proxyInfo,
request: crawlingContext.request,
log: crawlingContext.log,
- querySelector: (selector) => $(selector) as Cheerio<Element>,
- enqueueLinks: async (
+ async querySelector(selector, _timeoutMs?: number) {
+ return $(selector) as Cheerio<Element>;
+ },
+ async waitForSelector(selector, _timeoutMs?: number) {
+ if ($(selector).get().length === 0) {
+ throw new Error(`Selector '${selector}' not found.`);
+ }
+ },
+ async parseWithCheerio(selector?: string, _timeoutMs?: number): Promise<CheerioRoot> {
+ if (selector && $(selector).get().length === 0) {
+ throw new Error(`Selector '${selector}' not found.`);
+ }
+
+ return $;
+ },
+ async enqueueLinks(
options: Parameters<RestrictedCrawlingContext['enqueueLinks']>[0] = {},
- ) => {
+ ) {
const urls = extractUrlsFromCheerio(
$,
options.selector,
diff --git a/packages/playwright-crawler/src/internals/utils/playwright-utils.ts b/packages/playwright-crawler/src/internals/utils/playwright-utils.ts
index 2b8860af5..399cbd9a0 100644
--- a/packages/playwright-crawler/src/internals/utils/playwright-utils.ts
+++ b/packages/playwright-crawler/src/internals/utils/playwright-utils.ts
@@ -690,18 +690,34 @@ export interface PlaywrightContextUtils {
*/
blockRequests(options?: BlockRequestsOptions): Promise<void>;
+ /**
+ * Wait for an element matching the selector to appear.
+ * Timeout defaults to 5s.
+ *
+ * **Example usage:**
+ * ```ts
+ * async requestHandler({ waitForSelector, parseWithCheerio }) {
+ * await waitForSelector('article h1');
+ * const $ = await parseWithCheerio();
+ * const title = $('title').text();
+ * });
+ * ```
+ */
+ waitForSelector(selector: string, timeoutMs?: number): Promise<void>;
+
/**
* Returns Cheerio handle for `page.content()`, allowing to work with the data same way as with {@apilink CheerioCrawler}.
+ * When provided with the `selector` argument, it waits for it to be available first.
*
* **Example usage:**
- * ```javascript
+ * ```ts
* async requestHandler({ parseWithCheerio }) {
* const $ = await parseWithCheerio();
* const title = $('title').text();
* });
* ```
*/
- parseWithCheerio(): Promise<CheerioRoot>;
+ parseWithCheerio(selector?: string, timeoutMs?: number): Promise<CheerioRoot>;
/**
* Scrolls to the bottom of a page, or until it times out.
@@ -811,7 +827,17 @@ export function registerUtilsToContext(
await injectJQuery(context.page, { surviveNavigations: false });
};
context.blockRequests = async (options?: BlockRequestsOptions) => blockRequests(context.page, options);
- context.parseWithCheerio = async () => parseWithCheerio(context.page, crawlerOptions.ignoreShadowRoots);
+ context.waitForSelector = async (selector: string, timeoutMs?: number) => {
+ const locator = context.page.locator(selector).first();
+ await locator.waitFor({ timeout: timeoutMs, state: 'attached' });
+ };
+ context.parseWithCheerio = async (selector?: string, timeoutMs?: number) => {
+ if (selector) {
+ await context.waitForSelector(selector, timeoutMs);
+ }
+
+ return parseWithCheerio(context.page, crawlerOptions.ignoreShadowRoots);
+ };
context.infiniteScroll = async (options?: InfiniteScrollOptions) => infiniteScroll(context.page, options);
context.saveSnapshot = async (options?: SaveSnapshotOptions) =>
saveSnapshot(context.page, { ...options, config: context.crawler.config });
diff --git a/packages/puppeteer-crawler/src/internals/utils/puppeteer_utils.ts b/packages/puppeteer-crawler/src/internals/utils/puppeteer_utils.ts
index 0879f0168..143f02c5a 100644
--- a/packages/puppeteer-crawler/src/internals/utils/puppeteer_utils.ts
+++ b/packages/puppeteer-crawler/src/internals/utils/puppeteer_utils.ts
@@ -780,18 +780,34 @@ export interface PuppeteerContextUtils {
*/
injectJQuery(): Promise<unknown>;
+ /**
+ * Wait for an element matching the selector to appear.
+ * Timeout defaults to 5s.
+ *
+ * **Example usage:**
+ * ```ts
+ * async requestHandler({ waitForSelector, parseWithCheerio }) {
+ * await waitForSelector('article h1');
+ * const $ = await parseWithCheerio();
+ * const title = $('title').text();
+ * });
+ * ```
+ */
+ waitForSelector(selector: string, timeoutMs?: number): Promise<void>;
+
/**
* Returns Cheerio handle for `page.content()`, allowing to work with the data same way as with {@apilink CheerioCrawler}.
+ * When provided with the `selector` argument, it waits for it to be available first.
*
* **Example usage:**
- * ```javascript
+ * ```ts
* async requestHandler({ parseWithCheerio }) {
* const $ = await parseWithCheerio();
* const title = $('title').text();
* });
* ```
*/
- parseWithCheerio(): Promise<CheerioRoot>;
+ parseWithCheerio(selector?: string, timeoutMs?: number): Promise<CheerioRoot>;
/**
* The function finds elements matching a specific CSS selector in a Puppeteer page,
@@ -1022,7 +1038,16 @@ export function registerUtilsToContext(
}
await injectJQuery(context.page, { surviveNavigations: false });
};
- context.parseWithCheerio = async () => parseWithCheerio(context.page, crawlerOptions.ignoreShadowRoots);
+ context.waitForSelector = async (selector: string, timeoutMs = 5_000) => {
+ await context.page.waitForSelector(selector, { timeout: timeoutMs });
+ };
+ context.parseWithCheerio = async (selector?: string, timeoutMs = 5_000) => {
+ if (selector) {
+ await context.waitForSelector(selector, timeoutMs);
+ }
+
+ return parseWithCheerio(context.page, crawlerOptions.ignoreShadowRoots);
+ };
context.enqueueLinksByClickingElements = async (
options: Omit<EnqueueLinksByClickingElementsOptions, 'page' | 'requestQueue'>,
) =>
diff --git a/scripts/typescript_fixes.mjs b/scripts/typescript_fixes.mjs
index d8b9ca56e..dcf1eef65 100644
--- a/scripts/typescript_fixes.mjs
+++ b/scripts/typescript_fixes.mjs
@@ -23,6 +23,8 @@ for (const filepath of files) {
line.match(/: Puppeteer\.\w+/) ||
// don't ask me why, but this one is needed too ¯\_(ツ)_/¯
line.match(/^export interface (PlaywrightHook|PuppeteerHook)/) ||
+ // adaptive crawler needs router override that is incompatible with the base type
+ line.match(/readonly router: RouterHandler<AdaptivePlaywrightCrawlerContext>/) ||
// /// <reference types="something" /> from newer nodenext resolutions
line.match(/^\/\/\/ <reference types="[^"]+" \/>/) ||
// import("something") from compatibility with ES2022 module -.-
diff --git a/test/core/crawlers/adaptive_playwright_crawler.test.ts b/test/core/crawlers/adaptive_playwright_crawler.test.ts
index a563ca167..2fa4206b0 100644
--- a/test/core/crawlers/adaptive_playwright_crawler.test.ts
+++ b/test/core/crawlers/adaptive_playwright_crawler.test.ts
@@ -112,9 +112,10 @@ describe('AdaptivePlaywrightCrawler', () => {
const url = new URL(`http://${HOSTNAME}:${port}${path}`);
const requestHandler: AdaptivePlaywrightCrawlerOptions['requestHandler'] = vi.fn(
- async ({ pushData, querySelector }) => {
+ async ({ pushData, parseWithCheerio }) => {
+ const $ = await parseWithCheerio('h1');
await pushData({
- heading: (await querySelector('h1')).text(),
+ heading: $('h1').text(),
});
},
);
diff --git a/test/core/crawlers/http_crawler.test.ts b/test/core/crawlers/http_crawler.test.ts
index 513c8bbf3..a639722fa 100644
--- a/test/core/crawlers/http_crawler.test.ts
+++ b/test/core/crawlers/http_crawler.test.ts
@@ -114,7 +114,7 @@ test('parseWithCheerio works', async () => {
const crawler = new HttpCrawler({
maxRequestRetries: 0,
requestHandler: async ({ parseWithCheerio }) => {
- const $ = await parseWithCheerio();
+ const $ = await parseWithCheerio('title');
results.push($('title').text());
},
});
diff --git a/test/e2e/playwright-enqueue-links-base/actor/main.js b/test/e2e/playwright-enqueue-links-base/actor/main.js
index 0a2c7a319..7364d2bb9 100644
--- a/test/e2e/playwright-enqueue-links-base/actor/main.js
+++ b/test/e2e/playwright-enqueue-links-base/actor/main.js
@@ -16,7 +16,7 @@ await Actor.main(async () => {
async requestHandler({ parseWithCheerio, enqueueLinks, request, log }) {
const { url, loadedUrl } = request;
- const $ = await parseWithCheerio();
+ const $ = await parseWithCheerio('title', 1_000);
const pageTitle = $('title').first().text();
log.info(`URL: ${url}; LOADED_URL: ${loadedUrl}; TITLE: ${pageTitle}`);
await Dataset.pushData({ url, loadedUrl, pageTitle });