Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
docs: refactor crawling context helper methods to have them properly documented (#1550)
Martin Adámek35b717bf7a67
diff --git a/packages/basic-crawler/src/internals/basic-crawler.ts b/packages/basic-crawler/src/internals/basic-crawler.tsindex 1debf5a29..17654dabf 100644--- a/packages/basic-crawler/src/internals/basic-crawler.ts+++ b/packages/basic-crawler/src/internals/basic-crawler.ts@@ -35,20 +35,15 @@ import {validators,RetryRequestError,} from '@crawlee/core';-import type { Method, OptionsInit, Response as GotResponse } from 'got-scraping';+import type { Method, OptionsInit } from 'got-scraping';import { gotScraping } from 'got-scraping';-import type { ProcessedRequest, Dictionary, Awaitable, BatchAddRequestsResult } from '@crawlee/types';+import type { ProcessedRequest, Dictionary, Awaitable } from '@crawlee/types';import { chunk, sleep } from '@crawlee/utils';import ow, { ArgumentError } from 'ow';-export interface BasicCrawlingContext<UserData extends Dictionary = Dictionary> extends CrawlingContext<UserData> {- crawler: BasicCrawler;- enqueueLinks: (options: BasicCrawlerEnqueueLinksOptions) => Promise<BatchAddRequestsResult>;- sendRequest: (overrideOptions?: Partial<OptionsInit>) => Promise<GotResponse<string>>;-}--/** @internal */-export interface BasicCrawlerEnqueueLinksOptions extends Omit<EnqueueLinksOptions, 'requestQueue'> {}+export interface BasicCrawlingContext<+ UserData extends Dictionary = Dictionary,+> extends CrawlingContext<BasicCrawler, UserData> {}/*** Since there's no set number of seconds before the container is terminated after@@ -856,10 +851,11 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContextlog: this.log,request,session,- enqueueLinks: async (enqueueOptions: BasicCrawlerEnqueueLinksOptions) => {+ enqueueLinks: async (enqueueOptions: Partial<EnqueueLinksOptions>) => {return enqueueLinks({- ...enqueueOptions,+ // specify the RQ first to allow overriding itrequestQueue: await this.getRequestQueue(),+ ...enqueueOptions,});},sendRequest: async (overrideOptions?: OptionsInit) => {diff --git a/packages/browser-crawler/src/internals/browser-crawler.ts b/packages/browser-crawler/src/internals/browser-crawler.tsindex 8e2f0c359..e9c6d8777 100644--- a/packages/browser-crawler/src/internals/browser-crawler.ts+++ b/packages/browser-crawler/src/internals/browser-crawler.ts@@ -1,6 +1,5 @@import { addTimeoutToPromise, tryCancel } from '@apify/timeout';import type {- EnqueueLinksOptions,CrawlingContext,ProxyConfiguration,ProxyInfo,@@ -11,6 +10,7 @@ import type {Dictionary,RequestHandler,ErrorHandler,+ EnqueueLinksOptions,} from '@crawlee/basic';import {cookieStringToToughCookie,@@ -33,30 +33,25 @@ import type {LaunchContext,} from '@crawlee/browser-pool';import { BROWSER_CONTROLLER_EVENTS, BrowserPool } from '@crawlee/browser-pool';-import type { GotOptionsInit, Response as GotResponse } from 'got-scraping';import ow from 'ow';-import type { BatchAddRequestsResult, Cookie as CookieObject } from '@crawlee/types';+import type { Cookie as CookieObject } from '@crawlee/types';import type { BrowserLaunchContext } from './browser-launcher';export interface BrowserCrawlingContext<+ Crawler = unknown,Page extends CommonPage = CommonPage,Response = Dictionary,ProvidedController = BrowserController,UserData extends Dictionary = Dictionary,-> extends CrawlingContext<UserData> {+> extends CrawlingContext<Crawler, UserData> {browserController: ProvidedController;page: Page;response?: Response;- crawler: BrowserCrawler;- enqueueLinks: (options?: BrowserCrawlerEnqueueLinksOptions) => Promise<BatchAddRequestsResult>;- sendRequest: (overrideOptions?: Partial<GotOptionsInit>) => Promise<GotResponse<string>>;}export type BrowserRequestHandler<Context extends BrowserCrawlingContext = BrowserCrawlingContext> = RequestHandler<Context>;-export type BrowserErrorHandler<Context extends BrowserCrawlingContext= BrowserCrawlingContext>= ErrorHandler<Context>;--export interface BrowserCrawlerEnqueueLinksOptions extends Omit<EnqueueLinksOptions, 'requestQueue' | 'urls'> {}+export type BrowserErrorHandler<Context extends BrowserCrawlingContext = BrowserCrawlingContext> = ErrorHandler<Context>;export type BrowserHook<Context = BrowserCrawlingContext,@@ -660,7 +655,7 @@ export abstract class BrowserCrawler</** @internal */interface EnqueueLinksInternalOptions {- options?: BrowserCrawlerEnqueueLinksOptions;+ options?: Partial<EnqueueLinksOptions>;page: CommonPage;requestQueue: RequestQueue;originalRequestUrl: string;diff --git a/packages/cheerio-crawler/src/internals/cheerio-crawler.ts b/packages/cheerio-crawler/src/internals/cheerio-crawler.tsindex 6034ccbf2..c30c72056 100644--- a/packages/cheerio-crawler/src/internals/cheerio-crawler.ts+++ b/packages/cheerio-crawler/src/internals/cheerio-crawler.ts@@ -9,7 +9,7 @@ import type {Configuration,} from '@crawlee/http';import { HttpCrawler, enqueueLinks, Router, resolveBaseUrlForEnqueueLinksFiltering } from '@crawlee/http';-import type { BatchAddRequestsResult, Dictionary } from '@crawlee/types';+import type { Dictionary } from '@crawlee/types';import type { CheerioOptions } from 'cheerio';import * as cheerio from 'cheerio';import { DomHandler } from 'htmlparser2';@@ -40,15 +40,12 @@ export interface CheerioCrawlingContext<* Cheerio is available only for HTML and XML content types.*/$: cheerio.CheerioAPI;-- enqueueLinks: (options?: CheerioCrawlerEnqueueLinksOptions) => Promise<BatchAddRequestsResult>;}export type CheerioRequestHandler<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawlerJSONData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler> = RequestHandler<CheerioCrawlingContext<UserData, JSONData>>;-export interface CheerioCrawlerEnqueueLinksOptions extends Omit<EnqueueLinksOptions, 'urls' | 'requestQueue'> {}/*** Provides a framework for the parallel crawling of web pages using plain HTTP requests and@@ -153,7 +150,7 @@ export class CheerioCrawler extends HttpCrawler<CheerioCrawlingContext> {get body() {return isXml ? $!.xml() : $!.html({ decodeEntities: false });},- enqueueLinks: async (enqueueOptions?: CheerioCrawlerEnqueueLinksOptions) => {+ enqueueLinks: async (enqueueOptions?: EnqueueLinksOptions) => {return cheerioCrawlerEnqueueLinks({options: enqueueOptions,$,@@ -181,7 +178,7 @@ export class CheerioCrawler extends HttpCrawler<CheerioCrawlingContext> {}interface EnqueueLinksInternalOptions {- options?: CheerioCrawlerEnqueueLinksOptions;+ options?: Partial<EnqueueLinksOptions>;$: cheerio.CheerioAPI | null;requestQueue: RequestQueue;originalRequestUrl: string;diff --git a/packages/core/src/autoscaling/system_status.ts b/packages/core/src/autoscaling/system_status.tsindex 06ce3bbbe..e8f9dbb7c 100644--- a/packages/core/src/autoscaling/system_status.ts+++ b/packages/core/src/autoscaling/system_status.ts@@ -251,6 +251,14 @@ export class SystemStatus {* set to true if at least the ratio of snapshots in the sample are overloaded.*/protected _isSampleOverloaded<T extends { createdAt: Date; isOverloaded: boolean }>(sample: T[], ratio: number): ClientInfo {+ if (sample.length === 0) {+ return {+ isOverloaded: false,+ limitRatio: ratio,+ actualRatio: 0,+ };+ }+const weights: number[] = [];const values: number[] = [];@@ -262,7 +270,7 @@ export class SystemStatus {values.push(+current.isOverloaded);}- const wAvg = weightedAvg(values, weights);+ const wAvg = sample.length === 1 ? +sample[0].isOverloaded : weightedAvg(values, weights);return {isOverloaded: wAvg > ratio,diff --git a/packages/core/src/crawlers/crawler_commons.ts b/packages/core/src/crawlers/crawler_commons.tsindex 2fbf81f9d..73329a058 100644--- a/packages/core/src/crawlers/crawler_commons.ts+++ b/packages/core/src/crawlers/crawler_commons.ts@@ -1,10 +1,13 @@-import type { Dictionary } from '@crawlee/types';+import type { Dictionary, BatchAddRequestsResult } from '@crawlee/types';+import type { Response as GotResponse, OptionsInit } from 'got-scraping';++import type { EnqueueLinksOptions } from '../enqueue_links/enqueue_links';import type { Log } from '../log';import type { ProxyInfo } from '../proxy_configuration';import type { Request } from '../request';import type { Session } from '../session_pool/session';-export interface CrawlingContext<UserData extends Dictionary = Dictionary> extends Record<PropertyKey, unknown> {+export interface CrawlingContext<Crawler = unknown, UserData extends Dictionary = Dictionary> extends Record<PropertyKey, unknown> {id: string;/*** The original {@apilink Request} object.@@ -18,4 +21,51 @@ export interface CrawlingContext<UserData extends Dictionary = Dictionary> exten*/proxyInfo?: ProxyInfo;log: Log;+ crawler: Crawler;++ /**+ * This function automatically finds and enqueues links from the current page, adding them to the {@apilink RequestQueue}+ * currently used by the crawler.+ *+ * Optionally, the function allows you to filter the target links' URLs using an array of globs or regular expressions+ * and override settings of the enqueued {@apilink Request} objects.+ *+ * Check out the [Crawl a website with relative links](https://crawlee.dev/docs/examples/crawl-relative-links) example+ * for more details regarding its usage.+ *+ * **Example usage**+ *+ * ```ts+ * async requestHandler({ enqueueLinks }) {+ * await enqueueLinks({+ * globs: [+ * 'https://www.example.com/handbags/*',+ * ],+ * });+ * },+ * ```+ *+ * @param options All `enqueueLinks()` parameters are passed via an options object.+ * @returns Promise that resolves to {@apilink BatchAddRequestsResult} object.+ */+ enqueueLinks(options: EnqueueLinksOptions): Promise<BatchAddRequestsResult>;++ /**+ * Fires HTTP request via [`got-scraping`](https://crawlee.dev/docs/guides/got-scraping), allowing to override the request+ * options on the fly.+ *+ * This is handy when you work with a browser crawler but want to execute some requests outside it (e.g. API requests).+ * Check the [Skipping navigations for certain requests](https://crawlee.dev/docs/examples/skip-navigation) example for+ * more detailed explanation of how to do that.+ *+ * ```ts+ * async requestHandler({ sendRequest }) {+ * const { body } = await sendRequest({+ * // override headers only+ * headers: { ... },+ * });+ * },+ * ```+ */+ sendRequest<Response = string>(overrideOptions?: Partial<OptionsInit>): Promise<GotResponse<Response>>;}diff --git a/packages/core/src/enqueue_links/enqueue_links.ts b/packages/core/src/enqueue_links/enqueue_links.tsindex ad72f0930..15fef676d 100644--- a/packages/core/src/enqueue_links/enqueue_links.ts+++ b/packages/core/src/enqueue_links/enqueue_links.ts@@ -19,7 +19,7 @@ export interface EnqueueLinksOptions {limit?: number;/** An array of URLs to enqueue. */- urls: string[];+ urls?: string[];/** A request queue to which the URLs will be enqueued. */requestQueue: RequestQueue;diff --git a/packages/http-crawler/src/internals/http-crawler.ts b/packages/http-crawler/src/internals/http-crawler.tsindex 31c4cd172..ef13072a8 100644--- a/packages/http-crawler/src/internals/http-crawler.ts+++ b/packages/http-crawler/src/internals/http-crawler.ts@@ -23,7 +23,7 @@ import type { Awaitable, Dictionary } from '@crawlee/types';import type { RequestLike, ResponseLike } from 'content-type';import contentTypeParser from 'content-type';import mime from 'mime-types';-import type { OptionsInit, Method, Request as GotRequest, Response as GotResponse, GotOptionsInit, Options } from 'got-scraping';+import type { OptionsInit, Method, Request as GotRequest, Options } from 'got-scraping';import { gotScraping, TimeoutError } from 'got-scraping';import type { JsonValue } from 'type-fest';import { extname } from 'node:path';@@ -166,7 +166,7 @@ export interface InternalHttpCrawlingContext<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawlerJSONData extends JsonValue = any, // with default to Dictionary we cant use a typed router in untyped crawlerCrawler = HttpCrawler<any>- > extends CrawlingContext<UserData> {+ > extends CrawlingContext<Crawler, UserData> {/*** The request body of the web page.* The type depends on the `Content-Type` header of the web page:@@ -184,9 +184,7 @@ export interface InternalHttpCrawlingContext<* Parsed `Content-Type header: { type, encoding }`.*/contentType: { type: string; encoding: BufferEncoding };- crawler: Crawler;response: IncomingMessage;- sendRequest: (overrideOptions?: Partial<GotOptionsInit>) => Promise<GotResponse<string>>;}export interface HttpCrawlingContext<UserData extends Dictionary = any, JSONData extends JsonValue = any>diff --git a/packages/jsdom-crawler/src/internals/jsdom-crawler.ts b/packages/jsdom-crawler/src/internals/jsdom-crawler.tsindex 3b7866d47..567947f78 100644--- a/packages/jsdom-crawler/src/internals/jsdom-crawler.ts+++ b/packages/jsdom-crawler/src/internals/jsdom-crawler.ts@@ -35,14 +35,13 @@ export interface JSDOMCrawlingContext<> extends InternalHttpCrawlingContext<UserData, JSONData, JSDOMCrawler> {window: DOMWindow;- enqueueLinks: (options?: JSDOMCrawlerEnqueueLinksOptions) => Promise<BatchAddRequestsResult>;+ enqueueLinks: (options?: EnqueueLinksOptions) => Promise<BatchAddRequestsResult>;}export type JSDOMRequestHandler<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawlerJSONData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler> = RequestHandler<JSDOMCrawlingContext<UserData, JSONData>>;-export interface JSDOMCrawlerEnqueueLinksOptions extends Omit<EnqueueLinksOptions, 'urls' | 'requestQueue'> {}/*** Provides a framework for the parallel crawling of web pages using plain HTTP requests and@@ -125,7 +124,7 @@ export class JSDOMCrawler extends HttpCrawler<JSDOMCrawlingContext> {get body() {return window.document.documentElement.outerHTML;},- enqueueLinks: async (enqueueOptions?: JSDOMCrawlerEnqueueLinksOptions) => {+ enqueueLinks: async (enqueueOptions?: EnqueueLinksOptions) => {return domCrawlerEnqueueLinks({options: enqueueOptions,window,@@ -139,7 +138,7 @@ export class JSDOMCrawler extends HttpCrawler<JSDOMCrawlingContext> {}interface EnqueueLinksInternalOptions {- options?: JSDOMCrawlerEnqueueLinksOptions;+ options?: Partial<EnqueueLinksOptions>;window: DOMWindow | null;requestQueue: RequestQueue;originalRequestUrl: string;diff --git a/packages/playwright-crawler/src/internals/playwright-crawler.ts b/packages/playwright-crawler/src/internals/playwright-crawler.tsindex 39de3c5bd..01155059d 100644--- a/packages/playwright-crawler/src/internals/playwright-crawler.ts+++ b/packages/playwright-crawler/src/internals/playwright-crawler.ts@@ -10,7 +10,7 @@ import type { DirectNavigationOptions, PlaywrightContextUtils } from './utils/plimport { gotoExtended, registerUtilsToContext } from './utils/playwright-utils';export interface PlaywrightCrawlingContext<UserData extends Dictionary = Dictionary> extends- BrowserCrawlingContext<Page, Response, PlaywrightController, UserData>, PlaywrightContextUtils {}+ BrowserCrawlingContext<PlaywrightCrawler, Page, Response, PlaywrightController, UserData>, PlaywrightContextUtils {}export interface PlaywrightHook extends BrowserHook<PlaywrightCrawlingContext, PlaywrightGotoOptions> {}export interface PlaywrightRequestHandler extends BrowserRequestHandler<PlaywrightCrawlingContext> {}export type PlaywrightGotoOptions = Parameters<Page['goto']>[1];diff --git a/packages/playwright-crawler/src/internals/utils/playwright-utils.ts b/packages/playwright-crawler/src/internals/utils/playwright-utils.tsindex cb9b6c6ed..f6559ef53 100644--- a/packages/playwright-crawler/src/internals/utils/playwright-utils.ts+++ b/packages/playwright-crawler/src/internals/utils/playwright-utils.ts@@ -500,13 +500,148 @@ export async function parseWithCheerio(page: Page): Promise<CheerioRoot> {return cheerio.load(pageContent);}+/** @internal */export interface PlaywrightContextUtils {+ /**+ * Injects a JavaScript file into current `page`.+ * Unlike Playwright's `addScriptTag` function, this function works on pages+ * with arbitrary Cross-Origin Resource Sharing (CORS) policies.+ *+ * File contents are cached for up to 10 files to limit file system access.+ */injectFile(filePath: string, options?: InjectFileOptions): Promise<unknown>;++ /**+ * Injects the [jQuery](https://jquery.com/) library into current `page`.+ * jQuery is often useful for various web scraping and crawling tasks.+ * For example, it can help extract text from HTML elements using CSS selectors.+ *+ * Beware that the injected jQuery object will be set to the `window.$` variable and thus it might cause conflicts with+ * other libraries included by the page that use the same variable name (e.g. another version of jQuery).+ * This can affect functionality of page's scripts.+ *+ * The injected jQuery will survive page navigations and reloads.+ *+ * **Example usage:**+ * ```javascript+ * async requestHandler({ page, injectJQuery }) {+ * await injectJQuery();+ * const title = await page.evaluate(() => {+ * return $('head title').text();+ * });+ * });+ * ```+ *+ * Note that `injectJQuery()` does not affect the Playwright+ * [`page.$()`](https://playwright.dev/docs/api/class-page#page-query-selector)+ * function in any way.+ */injectJQuery(): Promise<unknown>;++ /**+ * Forces the Playwright browser tab to block loading URLs that match a provided pattern.+ * This is useful to speed up crawling of websites, since it reduces the amount+ * of data that needs to be downloaded from the web, but it may break some websites+ * or unexpectedly prevent loading of resources.+ *+ * By default, the function will block all URLs including the following patterns:+ *+ * ```json+ * [".css", ".jpg", ".jpeg", ".png", ".svg", ".gif", ".woff", ".pdf", ".zip"]+ * ```+ *+ * If you want to extend this list further, use the `extraUrlPatterns` option,+ * which will keep blocking the default patterns, as well as add your custom ones.+ * If you would like to block only specific patterns, use the `urlPatterns` option,+ * which will override the defaults and block only URLs with your custom patterns.+ *+ * This function does not use Playwright's request interception and therefore does not interfere+ * with browser cache. It's also faster than blocking requests using interception,+ * because the blocking happens directly in the browser without the round-trip to Node.js,+ * but it does not provide the extra benefits of request interception.+ *+ * The function will never block main document loads and their respective redirects.+ *+ * **Example usage**+ * ```javascript+ * preNavigationHooks: [+ * async ({ blockRequests }) => {+ * // Block all requests to URLs that include `adsbygoogle.js` and also all defaults.+ * await blockRequests({+ * extraUrlPatterns: ['adsbygoogle.js'],+ * }),+ * }),+ * ],+ * ```+ */blockRequests(options?: BlockRequestsOptions): Promise<void>;++ /**+ * Returns Cheerio handle for `page.content()`, allowing to work with the data same way as with {@apilink CheerioCrawler}.+ *+ * **Example usage:**+ * ```javascript+ * async requestHandler({ parseWithCheerio }) {+ * const $ = await parseWithCheerio();+ * const title = $('title').text();+ * });+ * ```+ */parseWithCheerio(): Promise<CheerioRoot>;++ /**+ * Scrolls to the bottom of a page, or until it times out.+ * Loads dynamic content when it hits the bottom of a page, and then continues scrolling.+ */infiniteScroll(options?: InfiniteScrollOptions): Promise<void>;++ /**+ * Saves a full screenshot and HTML of the current page into a Key-Value store.+ * @param [options]+ */saveSnapshot(options?: SaveSnapshotOptions): Promise<void>;++ /**+ * The function finds elements matching a specific CSS selector in a Playwright page,+ * clicks all those elements using a mouse move and a left mouse button click and intercepts+ * all the navigation requests that are subsequently produced by the page. The intercepted+ * requests, including their methods, headers and payloads are then enqueued to a provided+ * {@apilink RequestQueue}. This is useful to crawl JavaScript heavy pages where links are not available+ * in `href` elements, but rather navigations are triggered in click handlers.+ * If you're looking to find URLs in `href` attributes of the page, see {@apilink enqueueLinks}.+ *+ * Optionally, the function allows you to filter the target links' URLs using an array of {@apilink PseudoUrl} objects+ * and override settings of the enqueued {@apilink Request} objects.+ *+ * **IMPORTANT**: To be able to do this, this function uses various mutations on the page,+ * such as changing the Z-index of elements being clicked and their visibility. Therefore,+ * it is recommended to only use this function as the last operation in the page.+ *+ * **USING HEADFUL BROWSER**: When using a headful browser, this function will only be able to click elements+ * in the focused tab, effectively limiting concurrency to 1. In headless mode, full concurrency can be achieved.+ *+ * **PERFORMANCE**: Clicking elements with a mouse and intercepting requests is not a low level operation+ * that takes nanoseconds. It's not very CPU intensive, but it takes time. We strongly recommend limiting+ * the scope of the clicking as much as possible by using a specific selector that targets only the elements+ * that you assume or know will produce a navigation. You can certainly click everything by using+ * the `*` selector, but be prepared to wait minutes to get results on a large and complex page.+ *+ * **Example usage**+ *+ * ```javascript+ * async requestHandler({ enqueueLinksByClickingElements }) {+ * await enqueueLinksByClickingElements({+ * selector: 'a.product-detail',+ * globs: [+ * 'https://www.example.com/handbags/**'+ * 'https://www.example.com/purses/**'+ * ],+ * });+ * });+ * ```+ *+ * @returns Promise that resolves to {@apilink BatchAddRequestsResult} object.+ */enqueueLinksByClickingElements(options: Omit<EnqueueLinksByClickingElementsOptions, 'page' | 'requestQueue'>): Promise<BatchAddRequestsResult>;}diff --git a/packages/puppeteer-crawler/src/internals/puppeteer-crawler.ts b/packages/puppeteer-crawler/src/internals/puppeteer-crawler.tsindex 0a3bca586..f887bdd80 100644--- a/packages/puppeteer-crawler/src/internals/puppeteer-crawler.ts+++ b/packages/puppeteer-crawler/src/internals/puppeteer-crawler.ts@@ -15,7 +15,7 @@ import type { DirectNavigationOptions, PuppeteerContextUtils } from './utils/pupimport { gotoExtended, registerUtilsToContext } from './utils/puppeteer_utils';export interface PuppeteerCrawlingContext<UserData extends Dictionary = Dictionary> extends- BrowserCrawlingContext<Page, HTTPResponse, PuppeteerController, UserData>, PuppeteerContextUtils {}+ BrowserCrawlingContext<PuppeteerCrawler, Page, HTTPResponse, PuppeteerController, UserData>, PuppeteerContextUtils {}export interface PuppeteerHook extends BrowserHook<PuppeteerCrawlingContext, PuppeteerGoToOptions> {}export interface PuppeteerRequestHandler extends BrowserRequestHandler<PuppeteerCrawlingContext> {}export type PuppeteerGoToOptions = Parameters<Page['goto']>[1];diff --git a/packages/puppeteer-crawler/src/internals/utils/puppeteer_utils.ts b/packages/puppeteer-crawler/src/internals/utils/puppeteer_utils.tsindex 3a96e9f24..f8af5d368 100644--- a/packages/puppeteer-crawler/src/internals/utils/puppeteer_utils.ts+++ b/packages/puppeteer-crawler/src/internals/utils/puppeteer_utils.ts@@ -5,7 +5,7 @@* **Example usage:**** ```javascript- * import { launchPuppeteer, puppeteerUtils } from 'crawlee';+ * import { launchPuppeteer, utils } from 'crawlee';** // Open https://www.example.com in Puppeteer* const browser = await launchPuppeteer();@@ -13,7 +13,7 @@* await page.goto('https://www.example.com');** // Inject jQuery into a page- * await puppeteerUtils.injectJQuery(page);+ * await utils.puppeteer.injectJQuery(page);* ```* @module puppeteerUtils*/@@ -148,7 +148,7 @@ export async function injectFile(page: Page, filePath: string, options: InjectFi** **Example usage:*** ```javascript- * await puppeteerUtils.injectJQuery(page);+ * await utils.puppeteer.injectJQuery(page);* const title = await page.evaluate(() => {* return $('head title').text();* });@@ -170,7 +170,7 @@ export function injectJQuery(page: Page): Promise<unknown> {** **Example usage:*** ```javascript- * const $ = await puppeteerUtils.parseWithCheerio(page);+ * const $ = await utils.puppeteer.parseWithCheerio(page);* const title = $('title').text();* ```*@@ -208,13 +208,13 @@ export async function parseWithCheerio(page: Page): Promise<CheerioRoot> {** **Example usage*** ```javascript- * import { launchPuppeteer, puppeteerUtils } from 'crawlee';+ * import { launchPuppeteer, utils } from 'crawlee';** const browser = await launchPuppeteer();* const page = await browser.newPage();** // Block all requests to URLs that include `adsbygoogle.js` and also all defaults.- * await puppeteerUtils.blockRequests(page, {+ * await utils.puppeteer.blockRequests(page, {* extraUrlPatterns: ['adsbygoogle.js'],* });*@@ -372,7 +372,6 @@ export async function cacheResponses(page: Page, cache: Dictionary<Partial<Respo* Custom context may also be provided using the `context` parameter. To improve security,* make sure to only pass the really necessary objects to the context. Preferably making* secured copies beforehand.- **/export function compileScript(scriptString: string, context: Dictionary = Object.create(null)): CompiledScriptFunction {const funcString = `async ({ page, request }) => {${scriptString}}`;@@ -656,18 +655,256 @@ export async function saveSnapshot(page: Page, options: SaveSnapshotOptions = {}}}+/** @internal */export interface PuppeteerContextUtils {+ /**+ * Injects a JavaScript file into current `page`.+ * Unlike Puppeteer's `addScriptTag` function, this function works on pages+ * with arbitrary Cross-Origin Resource Sharing (CORS) policies.+ *+ * File contents are cached for up to 10 files to limit file system access.+ */injectFile(filePath: string, options?: InjectFileOptions): Promise<unknown>;++ /**+ * Injects the [jQuery](https://jquery.com/) library into current `page`.+ * jQuery is often useful for various web scraping and crawling tasks.+ * For example, it can help extract text from HTML elements using CSS selectors.+ *+ * Beware that the injected jQuery object will be set to the `window.$` variable and thus it might cause conflicts with+ * other libraries included by the page that use the same variable name (e.g. another version of jQuery).+ * This can affect functionality of page's scripts.+ *+ * The injected jQuery will survive page navigations and reloads.+ *+ * **Example usage:**+ * ```javascript+ * async requestHandler({ page, injectJQuery }) {+ * await injectJQuery();+ * const title = await page.evaluate(() => {+ * return $('head title').text();+ * });+ * });+ * ```+ *+ * Note that `injectJQuery()` does not affect the Puppeteer's+ * [`page.$()`](https://pptr.dev/api/puppeteer.page._/)+ * function in any way.+ */injectJQuery(): Promise<unknown>;++ /**+ * Returns Cheerio handle for `page.content()`, allowing to work with the data same way as with {@apilink CheerioCrawler}.+ *+ * **Example usage:**+ * ```javascript+ * async requestHandler({ parseWithCheerio }) {+ * const $ = await parseWithCheerio();+ * const title = $('title').text();+ * });+ * ```+ */parseWithCheerio(): Promise<CheerioRoot>;++ /**+ * The function finds elements matching a specific CSS selector in a Puppeteer page,+ * clicks all those elements using a mouse move and a left mouse button click and intercepts+ * all the navigation requests that are subsequently produced by the page. The intercepted+ * requests, including their methods, headers and payloads are then enqueued to a provided+ * {@apilink RequestQueue}. This is useful to crawl JavaScript heavy pages where links are not available+ * in `href` elements, but rather navigations are triggered in click handlers.+ * If you're looking to find URLs in `href` attributes of the page, see {@apilink enqueueLinks}.+ *+ * Optionally, the function allows you to filter the target links' URLs using an array of {@apilink PseudoUrl} objects+ * and override settings of the enqueued {@apilink Request} objects.+ *+ * **IMPORTANT**: To be able to do this, this function uses various mutations on the page,+ * such as changing the Z-index of elements being clicked and their visibility. Therefore,+ * it is recommended to only use this function as the last operation in the page.+ *+ * **USING HEADFUL BROWSER**: When using a headful browser, this function will only be able to click elements+ * in the focused tab, effectively limiting concurrency to 1. In headless mode, full concurrency can be achieved.+ *+ * **PERFORMANCE**: Clicking elements with a mouse and intercepting requests is not a low level operation+ * that takes nanoseconds. It's not very CPU intensive, but it takes time. We strongly recommend limiting+ * the scope of the clicking as much as possible by using a specific selector that targets only the elements+ * that you assume or know will produce a navigation. You can certainly click everything by using+ * the `*` selector, but be prepared to wait minutes to get results on a large and complex page.+ *+ * **Example usage**+ *+ * ```javascript+ * async requestHandler({ enqueueLinksByClickingElements }) {+ * await enqueueLinksByClickingElements({+ * selector: 'a.product-detail',+ * globs: [+ * 'https://www.example.com/handbags/**'+ * 'https://www.example.com/purses/**'+ * ],+ * });+ * });+ * ```+ *+ * @returns Promise that resolves to {@apilink BatchAddRequestsResult} object.+ */enqueueLinksByClickingElements(options: Omit<EnqueueLinksByClickingElementsOptions, 'page' | 'requestQueue'>): Promise<BatchAddRequestsResult>;++ /**+ * Forces the Puppeteer browser tab to block loading URLs that match a provided pattern.+ * This is useful to speed up crawling of websites, since it reduces the amount+ * of data that needs to be downloaded from the web, but it may break some websites+ * or unexpectedly prevent loading of resources.+ *+ * By default, the function will block all URLs including the following patterns:+ *+ * ```json+ * [".css", ".jpg", ".jpeg", ".png", ".svg", ".gif", ".woff", ".pdf", ".zip"]+ * ```+ *+ * If you want to extend this list further, use the `extraUrlPatterns` option,+ * which will keep blocking the default patterns, as well as add your custom ones.+ * If you would like to block only specific patterns, use the `urlPatterns` option,+ * which will override the defaults and block only URLs with your custom patterns.+ *+ * This function does not use Puppeteer's request interception and therefore does not interfere+ * with browser cache. It's also faster than blocking requests using interception,+ * because the blocking happens directly in the browser without the round-trip to Node.js,+ * but it does not provide the extra benefits of request interception.+ *+ * The function will never block main document loads and their respective redirects.+ *+ * **Example usage**+ * ```javascript+ * preNavigationHooks: [+ * async ({ blockRequests }) => {+ * // Block all requests to URLs that include `adsbygoogle.js` and also all defaults.+ * await blockRequests({+ * extraUrlPatterns: ['adsbygoogle.js'],+ * }),+ * }),+ * ],+ * ```+ */blockRequests(options?: BlockRequestsOptions): Promise<void>;++ /**+ * `blockResources()` has a high impact on performance in recent versions of Puppeteer.+ * Until this resolves, please use `utils.puppeteer.blockRequests()`.+ * @deprecated+ */blockResources(resourceTypes?: string[]): Promise<void>;++ /**+ * *NOTE:* In recent versions of Puppeteer using this function entirely disables browser cache which resolves in sub-optimal+ * performance. Until this resolves, we suggest just relying on the in-browser cache unless absolutely necessary.+ *+ * Enables caching of intercepted responses into a provided object. Automatically enables request interception in Puppeteer.+ * *IMPORTANT*: Caching responses stores them to memory, so too loose rules could cause memory leaks for longer running crawlers.+ * This issue should be resolved or atleast mitigated in future iterations of this feature.+ * @param cache+ * Object in which responses are stored+ * @param responseUrlRules+ * List of rules that are used to check if the response should be cached.+ * String rules are compared as page.url().includes(rule) while RegExp rules are evaluated as rule.test(page.url()).+ * @deprecated+ */cacheResponses(cache: Dictionary<Partial<ResponseForRequest>>, responseUrlRules: (string | RegExp)[]): Promise<void>;++ /**+ * Compiles a Puppeteer script into an async function that may be executed at any time+ * by providing it with the following object:+ * ```+ * {+ * page: Page,+ * request: Request,+ * }+ * ```+ * Where `page` is a Puppeteer [`Page`](https://pptr.dev/api/puppeteer.page)+ * and `request` is a {@apilink Request}.+ *+ * The function is compiled by using the `scriptString` parameter as the function's body,+ * so any limitations to function bodies apply. Return value of the compiled function+ * is the return value of the function body = the `scriptString` parameter.+ *+ * As a security measure, no globals such as `process` or `require` are accessible+ * from within the function body. Note that the function does not provide a safe+ * sandbox and even though globals are not easily accessible, malicious code may+ * still execute in the main process via prototype manipulation. Therefore you+ * should only use this function to execute sanitized or safe code.+ *+ * Custom context may also be provided using the `context` parameter. To improve security,+ * make sure to only pass the really necessary objects to the context. Preferably making+ * secured copies beforehand.+ */compileScript(scriptString: string, ctx?: Dictionary): CompiledScriptFunction;++ /**+ * Adds request interception handler in similar to `page.on('request', handler);` but in addition to that+ * supports multiple parallel handlers.+ *+ * All the handlers are executed sequentially in the order as they were added.+ * Each of the handlers must call one of `request.continue()`, `request.abort()` and `request.respond()`.+ * In addition to that any of the handlers may modify the request object (method, postData, headers)+ * by passing its overrides to `request.continue()`.+ * If multiple handlers modify same property then the last one wins. Headers are merged separately so you can+ * override only a value of specific header.+ *+ * If one the handlers calls `request.abort()` or `request.respond()` then request is not propagated further+ * to any of the remaining handlers.+ *+ *+ * **Example usage:**+ *+ * ```javascript+ * preNavigationHooks: [+ * async ({ addInterceptRequestHandler }) => {+ * // Replace images with placeholder.+ * await addInterceptRequestHandler((request) => {+ * if (request.resourceType() === 'image') {+ * return request.respond({+ * statusCode: 200,+ * contentType: 'image/jpeg',+ * body: placeholderImageBuffer,+ * });+ * }+ * return request.continue();+ * });+ *+ * // Abort all the scripts.+ * await addInterceptRequestHandler((request) => {+ * if (request.resourceType() === 'script') return request.abort();+ * return request.continue();+ * });+ *+ * // Change requests to post.+ * await addInterceptRequestHandler((request) => {+ * return request.continue({+ * method: 'POST',+ * });+ * });+ * }),+ * ],+ * ```+ * @param handler Request interception handler.+ */addInterceptRequestHandler(handler: InterceptHandler): Promise<void>;++ /**+ * Removes request interception handler for given page.+ *+ * @param handler Request interception handler.+ */removeInterceptRequestHandler(handler: InterceptHandler): Promise<void>;++ /**+ * Scrolls to the bottom of a page, or until it times out.+ * Loads dynamic content when it hits the bottom of a page, and then continues scrolling.+ */infiniteScroll(options?: InfiniteScrollOptions): Promise<void>;++ /**+ * Saves a full screenshot and HTML of the current page into a Key-Value store.+ */saveSnapshot(options?: SaveSnapshotOptions): Promise<void>;}diff --git a/test/core/crawlers/browser_crawler.test.ts b/test/core/crawlers/browser_crawler.test.tsindex dc066b54f..3322c6063 100644--- a/test/core/crawlers/browser_crawler.test.ts+++ b/test/core/crawlers/browser_crawler.test.ts@@ -5,8 +5,6 @@ import puppeteer from 'puppeteer';import log from '@apify/log';import type {BrowserCrawler,- BrowserCrawlingContext,- ProxyInfo,PuppeteerCrawlingContext,PuppeteerGoToOptions,PuppeteerRequestHandler,@@ -722,9 +720,9 @@ describe('BrowserCrawler', () => {});test('uses correct crawling context', async () => {- let prepareCrawlingContext: BrowserCrawlingContext;+ let prepareCrawlingContext: PuppeteerCrawlingContext;- const gotoFunction = async (crawlingContext: BrowserCrawlingContext) => {+ const gotoFunction = async (crawlingContext: PuppeteerCrawlingContext) => {prepareCrawlingContext = crawlingContext;expect(crawlingContext.request).toBeInstanceOf(Request);expect(crawlingContext.crawler.autoscaledPool).toBeInstanceOf(AutoscaledPool);@@ -732,7 +730,7 @@ describe('BrowserCrawler', () => {expect(typeof crawlingContext.page).toBe('object');};- const requestHandler = async (crawlingContext: BrowserCrawlingContext) => {+ const requestHandler = async (crawlingContext: PuppeteerCrawlingContext) => {expect(crawlingContext === prepareCrawlingContext).toEqual(true);expect(crawlingContext.request).toBeInstanceOf(Request);expect(crawlingContext.crawler.autoscaledPool).toBeInstanceOf(AutoscaledPool);@@ -744,14 +742,14 @@ describe('BrowserCrawler', () => {throw new Error('some error');};- const failedRequestHandler = async (crawlingContext: BrowserCrawlingContext, error: Error) => {+ const failedRequestHandler = async (crawlingContext: PuppeteerCrawlingContext, error: Error) => {expect(crawlingContext).toBe(prepareCrawlingContext);expect(crawlingContext.request).toBeInstanceOf(Request);expect(crawlingContext.crawler.autoscaledPool).toBeInstanceOf(AutoscaledPool);expect(crawlingContext.session).toBeInstanceOf(Session);expect(typeof crawlingContext.page).toBe('object');expect(crawlingContext.crawler).toBeInstanceOf(BrowserCrawlerTest);- expect((crawlingContext.crawler as BrowserCrawler).browserPool).toBeInstanceOf(BrowserPool);+ expect((crawlingContext.crawler).browserPool).toBeInstanceOf(BrowserPool);expect(crawlingContext.hasOwnProperty('response')).toBe(true);expect(crawlingContext.error).toBeInstanceOf(Error);diff --git a/test/core/crawlers/puppeteer_crawler.test.ts b/test/core/crawlers/puppeteer_crawler.test.tsindex 5c6250569..ef963d0cd 100644--- a/test/core/crawlers/puppeteer_crawler.test.ts+++ b/test/core/crawlers/puppeteer_crawler.test.ts@@ -1,7 +1,7 @@import log from '@apify/log';import type {+ PuppeteerCrawlingContext,PuppeteerGoToOptions,- PuppeteerRequestHandler,Request,} from '@crawlee/puppeteer';import {@@ -99,7 +99,7 @@ describe('PuppeteerCrawler', () => {const failed: Request[] = [];const asserts: boolean[] = [];const requestListLarge = await RequestList.open({ sources: sourcesLarge });- const requestHandler = async ({ page, request, response }: Parameters<PuppeteerRequestHandler>[0]) => {+ const requestHandler = async ({ page, request, response }: PuppeteerCrawlingContext) => {await page.waitForSelector('title');asserts.push(response.status() === 200);request.userData.title = await page.title();