Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
chore: update typescript and fix types in docs (#1241)
Szymon Marczakf741daba970c
diff --git a/src/apify.js b/src/apify.jsindex ef4c64a53..711acf07c 100644--- a/src/apify.js+++ b/src/apify.js@@ -1,5 +1,6 @@import ow from 'ow';import { ACT_JOB_STATUSES, ENV_VARS } from '@apify/consts';+// eslint-disable-next-line import/no-duplicatesimport { getEnv } from './actor';import { initializeEvents, stopEvents } from './events';import { StorageManager } from './storages/storage_manager';@@ -7,6 +8,7 @@ import { Dataset } from './storages/dataset';import { KeyValueStore, maybeStringify } from './storages/key_value_store';import { RequestList, REQUESTS_PERSISTENCE_KEY, STATE_PERSISTENCE_KEY } from './request_list';import { RequestQueue } from './storages/request_queue';+// eslint-disable-next-line import/no-duplicatesimport { SessionPool } from './session_pool/session_pool';import { ProxyConfiguration } from './proxy_configuration';import { addCharsetToContentType, logSystemInfo, printOutdatedSdkWarning, publicUtils, sleep } from './utils';@@ -20,6 +22,15 @@ import { enqueueLinks } from './enqueue_links/enqueue_links';import { requestAsBrowser } from './utils_request';import { ApifyCallError } from './errors';+// TYPE IMPORTS+/* eslint-disable no-unused-vars,import/named,import/no-duplicates,import/order */+import { ApifyClient } from 'apify-client';+import { UserFunc, AdhocWebhook, EventTypes, WebhookRun, ApifyEnv } from './actor';+import { ActorRun } from './typedefs';+import { RequestOptions } from './request';+import { SessionPoolOptions } from './session_pool/session_pool';+/* eslint-enable no-unused-vars,import/named,import/no-duplicates,import/order */+/*** `Apify` class serves as an alternative approach to the static helpers exported from the package. It allows to pass configuration* that will be used on the instance methods. Environment variables will have precedence over this configuration.diff --git a/src/autoscaling/autoscaled_pool.js b/src/autoscaling/autoscaled_pool.jsindex 142cf0d34..a88f90ee9 100644--- a/src/autoscaling/autoscaled_pool.js+++ b/src/autoscaling/autoscaled_pool.js@@ -3,7 +3,7 @@ import { betterSetInterval, betterClearInterval } from '@apify/utilities';import ow from 'ow';import Snapshotter, { SnapshotterOptions } from './snapshotter'; // eslint-disable-line import/named,no-unused-varsimport SystemStatus, { SystemStatusOptions } from './system_status'; // eslint-disable-line import/named,no-unused-vars-import defaultLog from '../utils_log';+import defaultLog, { Log } from '../utils_log'; // eslint-disable-line no-unused-vars/*** @typedef AutoscaledPoolOptions@@ -156,7 +156,7 @@ class AutoscaledPool {} = options;/**- * @type {defaultLog.Log}+ * @type {Log}*/this.log = log.child({ prefix: 'AutoscaledPool' });diff --git a/src/autoscaling/snapshotter.js b/src/autoscaling/snapshotter.jsindex 15362e7c5..f24c54b4a 100644--- a/src/autoscaling/snapshotter.js+++ b/src/autoscaling/snapshotter.js@@ -4,7 +4,7 @@ import { betterSetInterval, betterClearInterval } from '@apify/utilities';import { ACTOR_EVENT_NAMES, ENV_VARS } from '@apify/consts';import { getMemoryInfo, isAtHome, apifyClient } from '../utils';import events from '../events';-import defaultLog from '../utils_log';+import defaultLog, { Log } from '../utils_log'; // eslint-disable-line no-unused-varsconst RESERVE_MEMORY_RATIO = 0.5;const CLIENT_RATE_LIMIT_ERROR_RETRY_COUNT = 2;@@ -99,7 +99,7 @@ class Snapshotter {log = defaultLog,} = options;- /** @type {defaultLog.Log} */+ /** @type {Log} */this.log = log.child({ prefix: 'Snapshotter' });this.eventLoopSnapshotIntervalMillis = eventLoopSnapshotIntervalSecs * 1000;diff --git a/src/crawlers/basic_crawler.js b/src/crawlers/basic_crawler.jsindex 6bb6d8ab6..a0137d23d 100644--- a/src/crawlers/basic_crawler.js+++ b/src/crawlers/basic_crawler.js@@ -7,7 +7,7 @@ import AutoscaledPool from '../autoscaling/autoscaled_pool'; // eslint-disable-limport events from '../events';import { openSessionPool } from '../session_pool/session_pool'; // eslint-disable-line import/no-duplicatesimport Statistics from './statistics';-import defaultLog from '../utils_log';+import defaultLog from '../utils_log'; // eslint-disable-line import/no-duplicatesimport { validators } from '../validators';// TYPE IMPORTS@@ -20,6 +20,7 @@ import { RequestQueue } from '../storages/request_queue';import { QueueOperationInfo } from '../storages/request_queue';import { Session } from '../session_pool/session';import { SessionPoolOptions } from '../session_pool/session_pool';+import { Log } from '../utils_log';/* eslint-enable no-unused-vars,import/named,import/no-duplicates,import/order *//**@@ -254,7 +255,7 @@ export class BasicCrawler {}// assigning {} to the options as default break proper typing- /** @type {defaultLog.Log} */+ /** @type {Log} */this.log = log;this.requestList = requestList;this.requestQueue = requestQueue;@@ -684,7 +685,7 @@ export class BasicCrawler {}/**- * @param {Array<Hook>} hooks+ * @param {Array<any>} hooks* @param {*} args* @ignore* @protecteddiff --git a/src/crawlers/cheerio_crawler.js b/src/crawlers/cheerio_crawler.jsindex fb87a1b90..b2e13ffcf 100644--- a/src/crawlers/cheerio_crawler.js+++ b/src/crawlers/cheerio_crawler.js@@ -1,7 +1,7 @@/* eslint-disable class-methods-use-this */import { readStreamToString, concatStreamToBuffer } from '@apify/utilities';import { addTimeoutToPromise, tryCancel } from '@apify/timeout';-import cheerio from 'cheerio';+import cheerio from 'cheerio'; // eslint-disable-line import/no-duplicatesimport contentTypeParser from 'content-type';import { DomHandler } from 'htmlparser2';import { WritableStream } from 'htmlparser2/lib/WritableStream';@@ -11,7 +11,7 @@ import util from 'util';import { TimeoutError } from 'got-scraping';import { BASIC_CRAWLER_TIMEOUT_BUFFER_SECS } from '../constants';import { parseContentTypeFromResponse } from '../utils';-import { requestAsBrowser } from '../utils_request';+import { requestAsBrowser } from '../utils_request'; // eslint-disable-line import/no-duplicatesimport { diffCookies, mergeCookies } from './crawler_utils';import { BasicCrawler } from './basic_crawler'; // eslint-disable-line import/no-duplicatesimport CrawlerExtension from './crawler_extension';@@ -29,6 +29,9 @@ import { RequestQueue } from '../storages/request_queue';import { Session } from '../session_pool/session';import { SessionPoolOptions } from '../session_pool/session_pool';import { validators } from '../validators';+import { RequestAsBrowserOptions } from '../utils_request';+import { CheerioAPI } from 'cheerio';+import { Hook } from './browser_crawler';/* eslint-enable no-unused-vars,import/named,import/no-duplicates,import/order *//**@@ -471,12 +474,12 @@ class CheerioCrawler extends BasicCrawler {this.postResponseFunction = postResponseFunction;this.proxyConfiguration = proxyConfiguration;/**- * @type {Array<Hook>}+ * @type {Array<any>}* @ignore* */this.preNavigationHooks = preNavigationHooks;/**- * @type {Array<Hook>}+ * @type {Array<any>}* @ignore* */this.postNavigationHooks = [@@ -972,7 +975,7 @@ export default CheerioCrawler;/*** @typedef CheerioHandlePageInputs- * @property {cheerio.Root} $+ * @property {CheerioAPI} $* The [Cheerio](https://cheerio.js.org/) object with parsed HTML.* @property {(string|Buffer)} body* The request body of the web page.diff --git a/src/crawlers/puppeteer_crawler.js b/src/crawlers/puppeteer_crawler.jsindex 85928fc05..8de506ccb 100644--- a/src/crawlers/puppeteer_crawler.js+++ b/src/crawlers/puppeteer_crawler.js@@ -10,11 +10,12 @@ import BrowserCrawler, { BrowserCrawlingContext } from './browser_crawler';import { ProxyConfiguration } from '../proxy_configuration';import { SessionPoolOptions } from '../session_pool/session_pool';import { CrawlingContext } from './basic_crawler';-import { Page, DirectNavigationOptions } from 'puppeteer';+import { Page } from 'puppeteer';import { RequestList } from '../request_list';import { RequestQueue } from '../storages/request_queue';import { AutoscaledPoolOptions } from '../autoscaling/autoscaled_pool';import { BrowserPoolOptions } from 'browser-pool';+import { DirectNavigationOptions } from '../typedefs';/* eslint-enable no-unused-vars,import/named,import/no-duplicates,import/order *//**diff --git a/src/enqueue_links/enqueue_links.js b/src/enqueue_links/enqueue_links.jsindex 3cba07778..824a164fb 100644--- a/src/enqueue_links/enqueue_links.js+++ b/src/enqueue_links/enqueue_links.js@@ -13,6 +13,7 @@ import { RequestQueue, QueueOperationInfo } from '../storages/request_queue';import { RequestTransform } from './shared';import PseudoUrl from '../pseudo_url';import { validators } from '../validators';+import { CheerioAPI } from 'cheerio';/* eslint-enable no-unused-vars,import/named,import/no-duplicates,import/order *//**@@ -48,7 +49,7 @@ import { validators } from '../validators';* Either `page` or `$` option must be provided.* @param {Number} [options.limit]* Limit the count of actually enqueued URLs to this number. Useful for testing across the entire crawling scope.- * @param {(cheerio.Root|cheerio.Selector)} [options.$]+ * @param {CheerioAPI} [options.$]* [`Cheerio`](https://github.com/cheeriojs/cheerio) function with loaded HTML.* Either `page` or `$` option must be provided.* @param {RequestQueue} options.requestQueue@@ -159,7 +160,7 @@ export async function extractUrlsFromPage(page, selector) {/*** Extracts URLs from a given Cheerio object.*- * @param {(cheerio.Root|cheerio.Selector)} $+ * @param {CheerioAPI} $* @param {string} selector* @param {string} baseUrl* @return {string[]}diff --git a/src/playwright_utils.js b/src/playwright_utils.jsindex 7509f5956..b829ff538 100644--- a/src/playwright_utils.js+++ b/src/playwright_utils.js@@ -4,23 +4,11 @@ import { Page, Response } from 'playwright'; // eslint-disable-line no-unused-vaimport log from './utils_log';import { validators } from './validators';-import Request from './request'; // eslint-disable-line import/named,no-unused-vars+/* eslint-disable no-unused-vars,import/named,import/no-duplicates,import/order */+import { DirectNavigationOptions } from './typedefs';+/* eslint-enable no-unused-vars,import/named,import/no-duplicates,import/order */-/**- * @typedef {object} DirectNavigationOptions- * @property {number} [timeout]- * Maximum operation time in milliseconds, defaults to 30 seconds, pass `0` to disable timeout. The- * default value can be changed by using the browserContext.setDefaultNavigationTimeout(timeout),- * browserContext.setDefaultTimeout(timeout), page.setDefaultNavigationTimeout(timeout) or- * page.setDefaultTimeout(timeout) methods.- * @property {("domcontentloaded"|"load"|"networkidle")} [waitUntil]- * When to consider operation succeeded, defaults to `load`. Events can be either:- * - `'domcontentloaded'` - consider operation to be finished when the `DOMContentLoaded` event is fired.- * - `'load'` - consider operation to be finished when the `load` event is fired.- * - `'networkidle'` - consider operation to be finished when there are no network connections for at least `500` ms.- * @property {string} [referer]- * Referer header value. If provided it will take preference over the referer header value set by page.setExtraHTTPHeaders(headers).- */+import Request from './request'; // eslint-disable-line import/named,no-unused-vars/*** Extended version of Playwright's `page.goto()` allowing to perform requests with HTTP method other than GET,diff --git a/src/puppeteer_request_interception.js b/src/puppeteer_request_interception.jsindex f592b1d53..91e18e8fe 100644--- a/src/puppeteer_request_interception.js+++ b/src/puppeteer_request_interception.js@@ -1,7 +1,7 @@import { EventEmitter } from 'events';import ow from 'ow';import _ from 'underscore';-import { Request as PuppeteerRequest, Page } from 'puppeteer'; // eslint-disable-line no-unused-vars+import { HTTPRequest as PuppeteerRequest, Page } from 'puppeteer'; // eslint-disable-line no-unused-varsimport log from './utils_log';// We use weak maps here so that the content gets discarted after page gets closed.diff --git a/src/session_pool/session.js b/src/session_pool/session.jsindex f0c1e4bb9..1555045a2 100644--- a/src/session_pool/session.js+++ b/src/session_pool/session.js@@ -9,11 +9,14 @@ import defaultLog from '../utils_log';// TYPE IMPORTS/* eslint-disable no-unused-vars,import/named,import/no-duplicates,import/order,import/no-cycle */import { SessionPool } from './session_pool';-import { Cookie as PuppeteerCookie } from 'puppeteer';import { IncomingMessage } from 'http';-import { Response as PuppeteerResponse } from 'puppeteer';+import { HTTPResponse as PuppeteerResponse } from 'puppeteer';/* eslint-enable no-unused-vars,import/named,import/no-duplicates,import/order,import/no-cycle */+/**+ * @typedef {object} PuppeteerCookie+ */+// CONSTANTSconst DEFAULT_SESSION_MAX_AGE_SECS = 3000;diff --git a/src/typedefs.js b/src/typedefs.jsindex 3b2574669..75e64ffba 100644--- a/src/typedefs.js+++ b/src/typedefs.js@@ -84,4 +84,20 @@* ```*/+/**+ * @typedef {object} DirectNavigationOptions+ * @property {number} [timeout]+ * Maximum operation time in milliseconds, defaults to 30 seconds, pass `0` to disable timeout. The+ * default value can be changed by using the browserContext.setDefaultNavigationTimeout(timeout),+ * browserContext.setDefaultTimeout(timeout), page.setDefaultNavigationTimeout(timeout) or+ * page.setDefaultTimeout(timeout) methods.+ * @property {("domcontentloaded"|"load"|"networkidle")} [waitUntil]+ * When to consider operation succeeded, defaults to `load`. Events can be either:+ * - `'domcontentloaded'` - consider operation to be finished when the `DOMContentLoaded` event is fired.+ * - `'load'` - consider operation to be finished when the `load` event is fired.+ * - `'networkidle'` - consider operation to be finished when there are no network connections for at least `500` ms.+ * @property {string} [referer]+ * Referer header value. If provided it will take preference over the referer header value set by page.setExtraHTTPHeaders(headers).+ */+export {};diff --git a/src/utils.js b/src/utils.jsindex dd96fe1cc..e32352716 100644--- a/src/utils.js+++ b/src/utils.js@@ -3,6 +3,7 @@ import { execSync } from 'child_process';import { ApifyClient } from 'apify-client';import { version as apifyClientVersion } from 'apify-client/package.json';import { ACT_JOB_TERMINAL_STATUSES, ENV_VARS } from '@apify/consts';+// eslint-disable-next-line import/no-duplicatesimport cheerio from 'cheerio';import contentTypeParser from 'content-type';import fs from 'fs';@@ -19,13 +20,13 @@ import rimraf from 'rimraf';// TYPE IMPORTS/* eslint-disable no-unused-vars,import/named,import/no-duplicates,import/order */import { IncomingMessage } from 'http';-import { Response as PuppeteerResponse } from 'puppeteer';+import { HTTPResponse as PuppeteerResponse } from 'puppeteer';import { version as apifyVersion } from '../package.json';import log from './utils_log';import { requestAsBrowser } from './utils_request';import Request, { RequestOptions } from './request';import { ActorRun } from './typedefs';-+import { CheerioAPI } from 'cheerio';/* eslint-enable no-unused-vars,import/named,import/no-duplicates,import/order */const rimrafp = util.promisify(rimraf);@@ -473,7 +474,7 @@ const BLOCK_TAGS_REGEX = /^(p|h1|h2|h3|h4|h5|h6|ol|ul|li|pre|address|blockquote|* const html = '<html><body>Some text</body></html>';* const text = htmlToText(cheerio.load(html, { decodeEntities: true }));* ```- * @param {(string|cheerio.Root)} html HTML text or parsed HTML represented using a+ * @param {(string|CheerioAPI)} html HTML text or parsed HTML represented using a* [cheerio](https://www.npmjs.com/package/cheerio) function.* @return {string} Plain text* @memberOf utils@@ -488,7 +489,7 @@ const htmlToText = (html) => {// produces really text with a lot of HTML elements in it. Let's just deprecate this sort of usage,// and make the parameter "htmlOrCheerioElement"/**- * @type {cheerio.Root}+ * @type {CheerioAPI}* @ignore*/const $ = typeof html === 'function' ? html : cheerio.load(html, { decodeEntities: true });diff --git a/src/utils_request.js b/src/utils_request.jsindex a0c814812..6f1cc08c5 100644--- a/src/utils_request.js+++ b/src/utils_request.js@@ -7,7 +7,7 @@ import { IncomingMessage } from 'http';/* eslint-enable no-unused-vars,import/named,import/order *//**- * @typedef {(IncomingMessage & Readable & { body: string })} RequestAsBrowserResult+ * @typedef {(IncomingMessage & { body: string })} RequestAsBrowserResult*//**diff --git a/website/i18n/en.json b/website/i18n/en.jsonindex b717ac2aa..055c76cf9 100644--- a/website/i18n/en.json+++ b/website/i18n/en.json@@ -326,6 +326,9 @@"typedefs/proxy-info": {"title": "ProxyInfo"},+ "typedefs/puppeteer-cookie": {+ "title": "PuppeteerCookie"+ },"typedefs/puppeteer-crawler-options": {"title": "PuppeteerCrawlerOptions"},