Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Build documentation changes
Ondra Urbanbedc6d28f2f6
diff --git a/docs/typedefs/CheerioCrawlerOptions.md b/docs/typedefs/CheerioCrawlerOptions.mdindex 74d0d00f6..102c046c7 100644--- a/docs/typedefs/CheerioCrawlerOptions.md+++ b/docs/typedefs/CheerioCrawlerOptions.md@@ -164,8 +164,8 @@ The exceptions are logged to the request using the</tr><tr><td colspan="3"><p>An array of <a href="https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/MIME_types/Complete_list_of_MIME_types"- target="_blank">mime types</a> you want to process.- By default <code>text/html</code>, <code>application/xhtml+xml</code> mime types are supported.</p>+ target="_blank">MIME types</a> you want the crawler to load and process.+ By default, only <code>text/html</code> and <code>application/xhtml+xml</code> MIME types are supported.</p></td></tr><tr><td><code>[maxRequestRetries]</code></td><td><code>Number</code></td><td><code>3</code></td></tr>diff --git a/types/crawlers/cheerio_crawler.d.ts b/types/crawlers/cheerio_crawler.d.tsindex d4f25691d..0331793b6 100644--- a/types/crawlers/cheerio_crawler.d.ts+++ b/types/crawlers/cheerio_crawler.d.ts@@ -153,8 +153,8 @@ export type CheerioCrawlerOptions = {handleFailedRequestFunction?: HandleFailedRequest;/*** An array of <a href="https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/MIME_types/Complete_list_of_MIME_types"- * target="_blank">mime types</a> you want to process.- * By default `text/html`, `application/xhtml+xml` mime types are supported.+ * target="_blank">MIME types</a> you want the crawler to load and process.+ * By default, only `text/html` and `application/xhtml+xml` MIME types are supported.*/additionalMimeTypes?: string[];/**@@ -372,8 +372,8 @@ export type CheerioHandlePage = (inputs: CheerioHandlePageInputs) => Promise<voi* for the default implementation of this function.* @property {String[]} [additionalMimeTypes]* An array of <a href="https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/MIME_types/Complete_list_of_MIME_types"- * target="_blank">mime types</a> you want to process.- * By default `text/html`, `application/xhtml+xml` mime types are supported.+ * target="_blank">MIME types</a> you want the crawler to load and process.+ * By default, only `text/html` and `application/xhtml+xml` MIME types are supported.* @property {Number} [maxRequestRetries=3]* Indicates how many times the request is retried if either `requestFunction` or `handlePageFunction` fails.* @property {Number} [maxRequestsPerCrawl]diff --git a/types/session_pool/session_pool.d.ts b/types/session_pool/session_pool.d.tsindex 18edb7695..f0458804e 100644--- a/types/session_pool/session_pool.d.ts+++ b/types/session_pool/session_pool.d.ts@@ -171,12 +171,12 @@ export class SessionPool extends EventEmitter {addListener(event: string | symbol, listener: (...args: any[]) => void): SessionPool;on(event: string | symbol, listener: (...args: any[]) => void): SessionPool;once(event: string | symbol, listener: (...args: any[]) => void): SessionPool;- prependListener(event: string | symbol, listener: (...args: any[]) => void): SessionPool;- prependOnceListener(event: string | symbol, listener: (...args: any[]) => void): SessionPool;removeListener(event: string | symbol, listener: (...args: any[]) => void): SessionPool;off(event: string | symbol, listener: (...args: any[]) => void): SessionPool;removeAllListeners(event?: string | symbol): SessionPool;setMaxListeners(n: number): SessionPool;+ prependListener(event: string | symbol, listener: (...args: any[]) => void): SessionPool;+ prependOnceListener(event: string | symbol, listener: (...args: any[]) => void): SessionPool;}export function openSessionPool(sessionPoolOptions: SessionPoolOptions): Promise<SessionPool>;/**