Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

improve JSDoc for the SDK (#942)

Paulo Cesarda9bcf36b352
diff --git a/src/actor.js b/src/actor.js
index 27134b854..7ac19bcae 100644
--- a/src/actor.js
+++ b/src/actor.js
@@ -56,6 +56,10 @@ const tryParseDate = (str) => {
* in megabytes (APIFY_MEMORY_MBYTES)
*/
+/**
+ * @typedef {Array<('ACTOR.RUN.SUCCEEDED' | 'ACTOR.RUN.ABORTED' | 'ACTOR.RUN.CREATED' | 'ACTOR.RUN.FAILED' | 'ACTOR.RUN.TIMED_OUT')>} EventTypes
+ */
+
/**
* Returns a new {@link ApifyEnv} object which contains information parsed from all the `APIFY_XXX` environment variables.
*
@@ -218,6 +222,14 @@ export const main = (userFunc) => {
});
};
+/**
+ * @typedef AdhocWebhook
+ * @property {EventTypes} eventTypes
+ * @property {string} requestUrl
+ * @property {string} [idempotencyKey]
+ * @property {string} [payloadTemplate]
+ */
+
/**
* Runs an actor on the Apify platform using the current user account (determined by the `APIFY_TOKEN` environment variable),
* waits for the actor to finish and fetches its output.
@@ -248,11 +260,11 @@ export const main = (userFunc) => {
*
* @param {string} actId
* Allowed formats are `username/actor-name`, `userId/actor-name` or actor ID.
- * @param {object} [input]
+ * @param {Object<string, *>} [input]
* Input for the actor. If it is an object, it will be stringified to
* JSON and its content type set to `application/json; charset=utf-8`.
* Otherwise the `options.contentType` parameter must be provided.
- * @param {Object} [options={}]
+ * @param {object} [options]
* Object with the settings below:
* @param {string} [options.contentType]
* Content type for the `input`. If not specified,
@@ -280,7 +292,7 @@ export const main = (userFunc) => {
* @param {boolean} [options.disableBodyParser=false]
* If `true` then the function will not attempt to parse the
* actor's output and will return it in a raw `Buffer`.
- * @param {Array<object>} [options.webhooks] Specifies optional webhooks associated with the actor run, which can be used
+ * @param {Array<AdhocWebhook>} [options.webhooks] Specifies optional webhooks associated with the actor run, which can be used
* to receive a notification e.g. when the actor finished or failed, see
* [ad hook webhooks documentation](https://docs.apify.com/webhooks/ad-hoc-webhooks) for detailed description.
* @returns {Promise<ActorRun>}
@@ -381,11 +393,11 @@ export const call = async (actId, input, options = {}) => {
*
* @param {string} taskId
* Allowed formats are `username/task-name`, `userId/task-name` or task ID.
- * @param {object} [input]
+ * @param {Object<string, *>} [input]
* Input overrides for the actor task. If it is an object, it will be stringified to
* JSON and its content type set to `application/json; charset=utf-8`.
* Provided input will be merged with actor task input.
- * @param {Object} [options={}]
+ * @param {object} [options]
* Object with the settings below:
* @param {string} [options.token]
* User API token that is used to run the actor. By default, it is taken from the `APIFY_TOKEN` environment variable.
@@ -403,7 +415,7 @@ export const call = async (actId, input, options = {}) => {
* If the limit is reached, the returned promise is resolved to a run object that will have
* status `READY` or `RUNNING` and it will not contain the actor run output.
* If `waitSecs` is null or undefined, the function waits for the actor task to finish (default behavior).
- * @param {Array<object>} [options.webhooks] Specifies optional webhooks associated with the actor run, which can be used
+ * @param {Array<AdhocWebhook>} [options.webhooks] Specifies optional webhooks associated with the actor run, which can be used
* to receive a notification e.g. when the actor finished or failed, see
* [ad hook webhooks documentation](https://docs.apify.com/webhooks/ad-hoc-webhooks) for detailed description.
* @returns {Promise<ActorRun>}
@@ -481,11 +493,11 @@ function isRunUnsuccessful(status) {
*
* @param {string} targetActorId
* Either `username/actor-name` or actor ID of an actor to which we want to metamorph.
- * @param {object} [input]
+ * @param {Object<string, *>} [input]
* Input for the actor. If it is an object, it will be stringified to
* JSON and its content type set to `application/json; charset=utf-8`.
* Otherwise the `options.contentType` parameter must be provided.
- * @param {Object} [options={}]
+ * @param {object} [options]
* Object with the settings below:
* @param {string} [options.contentType]
* Content type for the `input`. If not specified,
@@ -532,6 +544,22 @@ export const metamorph = async (targetActorId, input, options = {}) => {
await sleep(customAfterSleepMillis || METAMORPH_AFTER_SLEEP_MILLIS);
};
+/**
+ * @typedef WebhookRun
+ * @property {string} id
+ * @property {string} createdAt
+ * @property {string} modifiedAt
+ * @property {string} userId
+ * @property {boolean} isAdHoc
+ * @property {EventTypes} eventTypes
+ * @property {*} condition
+ * @property {boolean} ignoreSslErrors
+ * @property {boolean} doNotRetry
+ * @property {string} requestUrl
+ * @property {string} payloadTemplate
+ * @property {*} lastDispatch
+ * @property {*} stats
+ */
/**
*
* Creates an ad-hoc webhook for the current actor run, which lets you receive a notification when the actor run finished or failed.
@@ -540,8 +568,8 @@ export const metamorph = async (targetActorId, input, options = {}) => {
* Note that webhooks are only supported for actors running on the Apify platform.
* In local environment, the function will print a warning and have no effect.
*
- * @param {Object} options
- * @param {string[]} options.eventTypes
+ * @param {object} options
+ * @param {EventTypes} options.eventTypes
* Array of event types, which you can set for actor run, see
* the [actor run events](https://docs.apify.com/webhooks/events#actor-run) in the Apify doc.
* @param {string} options.requestUrl
@@ -558,7 +586,7 @@ export const metamorph = async (targetActorId, input, options = {}) => {
* an actor restart or other situation that would cause the `addWebhook()` function to be called again.
* We suggest using the actor run ID as the idempotency key. You can get the run ID by calling
* {@link Apify#getEnv} function.
- * @return {Promise<object>} The return value is the Webhook object.
+ * @return {Promise<WebhookRun|undefined>} The return value is the Webhook object.
* For more information, see the [Get webhook](https://apify.com/docs/api/v2#/reference/webhooks/webhook-object/get-webhook) API endpoint.
*
* @memberof module:Apify
diff --git a/src/autoscaling/autoscaled_pool.js b/src/autoscaling/autoscaled_pool.js
index f6865c471..eb3bc9f31 100644
--- a/src/autoscaling/autoscaled_pool.js
+++ b/src/autoscaling/autoscaled_pool.js
@@ -6,17 +6,17 @@ import defaultLog from '../utils_log';
/**
* @typedef AutoscaledPoolOptions
- * @property {Function} runTaskFunction
+ * @property {Function} [runTaskFunction]
* A function that performs an asynchronous resource-intensive task.
* The function must either be labeled `async` or return a promise.
*
- * @property {Function} isTaskReadyFunction
+ * @property {Function} [isTaskReadyFunction]
* A function that indicates whether `runTaskFunction` should be called.
* This function is called every time there is free capacity for a new task and it should
* indicate whether it should start a new task or not by resolving to either `true` or `false`.
* Besides its obvious use, it is also useful for task throttling to save resources.
*
- * @property {Function} isFinishedFunction
+ * @property {Function} [isFinishedFunction]
* A function that is called only when there are no tasks to be processed.
* If it resolves to `true` then the pool's run finishes. Being called only
* when there are no tasks being processed means that as long as `isTaskReadyFunction()`
@@ -150,6 +150,9 @@ class AutoscaledPool {
log = defaultLog,
} = options;
+ /**
+ * @type {defaultLog.Log}
+ */
this.log = log.child({ prefix: 'AutoscaledPool' });
// Configurable properties.
@@ -361,6 +364,8 @@ class AutoscaledPool {
* It doesn't allow multiple concurrent runs of this method.
*
* @ignore
+ * @protected
+ * @internal
*/
async _maybeRunTask(intervalCallback) {
// Check if the function was invoked by the maybeRunInterval and use an empty function if not.
@@ -432,6 +437,8 @@ class AutoscaledPool {
* If the system IS overloaded and the settings allow it, it scales down.
*
* @ignore
+ * @protected
+ * @internal
*/
_autoscale(intervalCallback) {
// Don't scale if paused.
@@ -480,6 +487,8 @@ class AutoscaledPool {
*
* @param {Object} systemStatus for logging
* @ignore
+ * @protected
+ * @internal
*/
_scaleUp(systemStatus) {
const step = Math.ceil(this._desiredConcurrency * this.scaleUpStepRatio);
@@ -497,6 +506,8 @@ class AutoscaledPool {
*
* @param {Object} systemStatus for logging
* @ignore
+ * @protected
+ * @internal
*/
_scaleDown(systemStatus) {
const step = Math.ceil(this._desiredConcurrency * this.scaleUpStepRatio);
@@ -515,6 +526,8 @@ class AutoscaledPool {
* It doesn't allow multiple concurrent runs of this method.
*
* @ignore
+ * @protected
+ * @internal
*/
async _maybeFinish() {
if (this.queryingIsFinished) return;
@@ -539,6 +552,8 @@ class AutoscaledPool {
* Cleans up resources.
*
* @ignore
+ * @protected
+ * @internal
*/
async _destroy() {
this.resolve = null;
diff --git a/src/autoscaling/snapshotter.js b/src/autoscaling/snapshotter.js
index 4b56ca1ab..6857d5b51 100644
--- a/src/autoscaling/snapshotter.js
+++ b/src/autoscaling/snapshotter.js
@@ -99,6 +99,7 @@ class Snapshotter {
log = defaultLog,
} = options;
+ /** @type {defaultLog.Log} */
this.log = log.child({ prefix: 'Snapshotter' });
this.eventLoopSnapshotIntervalMillis = eventLoopSnapshotIntervalSecs * 1000;
@@ -209,6 +210,8 @@ class Snapshotter {
* @param {number} [sampleDurationMillis]
* @return {Array<*>}
* @ignore
+ * @protected
+ * @internal
*/
_getSample(snapshots, sampleDurationMillis) { // eslint-disable-line class-methods-use-this
if (!sampleDurationMillis) return snapshots;
@@ -231,6 +234,8 @@ class Snapshotter {
* using the Apify platform `systemInfo` event.
* @param {*} systemInfo
* @ignore
+ * @protected
+ * @internal
*/
_snapshotMemoryOnPlatform(systemInfo) {
const now = new Date();
@@ -252,6 +257,8 @@ class Snapshotter {
* @param {Function} intervalCallback
* @return {Promise<void>}
* @ignore
+ * @protected
+ * @internal
*/
async _snapshotMemoryOnLocal(intervalCallback) {
try {
@@ -279,6 +286,8 @@ class Snapshotter {
* Checks for critical memory overload and logs it to the console.
* @param {*} systemInfo
* @ignore
+ * @protected
+ * @internal
*/
_memoryOverloadWarning({ memCurrentBytes }) {
const now = new Date();
@@ -302,6 +311,8 @@ class Snapshotter {
* Creates a snapshot of current event loop delay.
* @param {Function} intervalCallback
* @ignore
+ * @protected
+ * @internal
*/
_snapshotEventLoop(intervalCallback) {
const now = new Date();
@@ -331,6 +342,8 @@ class Snapshotter {
* using the Apify platform `systemInfo` event.
* @param {Object} systemInfo
* @ignore
+ * @protected
+ * @internal
*/
_snapshotCpuOnPlatform(systemInfo) {
const { cpuCurrentUsage, isCpuOverloaded } = systemInfo;
@@ -349,6 +362,8 @@ class Snapshotter {
* using OS provided metrics.
* @param {Function} intervalCallback
* @ignore
+ * @protected
+ * @internal
*/
_snapshotCpuOnLocal(intervalCallback) {
const now = new Date();
@@ -377,6 +392,11 @@ class Snapshotter {
intervalCallback();
}
+ /**
+ * @ignore
+ * @protected
+ * @internal
+ */
_getCurrentCpuTicks() { // eslint-disable-line class-methods-use-this
const cpus = os.cpus();
return cpus.reduce((acc, cpu) => {
@@ -397,7 +417,9 @@ class Snapshotter {
* overloading.
*
* @param intervalCallback
- * @private
+ * @ignore
+ * @protected
+ * @internal
*/
_snapshotClient(intervalCallback) {
const now = new Date();
@@ -429,6 +451,8 @@ class Snapshotter {
* @param {Array<*>} snapshots
* @param {Date} now
* @ignore
+ * @protected
+ * @internal
*/
_pruneSnapshots(snapshots, now) {
let oldCount = 0;
@@ -443,6 +467,8 @@ class Snapshotter {
/**
* Calculate max memory for platform or local usage.
* @ignore
+ * @protected
+ * @internal
*/
async _ensureCorrectMaxMemory() {
if (this.maxMemoryBytes) return;
diff --git a/src/autoscaling/system_status.js b/src/autoscaling/system_status.js
index e1d82f94a..90f39d750 100644
--- a/src/autoscaling/system_status.js
+++ b/src/autoscaling/system_status.js
@@ -9,10 +9,10 @@ import { weightedAvg } from '../utils';
* @typedef SystemInfo
* @property {boolean} isSystemIdle
* If true, system is being overloaded.
- * @property {object} memInfo
+ * @property {Object<string, *>} memInfo
* Memory
- * @property {object} eventLoopInfo
- * @property {object} cpuInfo
+ * @property {Object<string, *>} eventLoopInfo
+ * @property {Object<string, *>} cpuInfo
*/
/**
@@ -136,6 +136,8 @@ class SystemStatus {
* @param {number} [sampleDurationMillis]
* @return {SystemInfo}
* @ignore
+ * @protected
+ * @internal
*/
_isSystemIdle(sampleDurationMillis) {
const memInfo = this._isMemoryOverloaded(sampleDurationMillis);
@@ -158,6 +160,8 @@ class SystemStatus {
* @param {number} sampleDurationMillis
* @return {object}
* @ignore
+ * @protected
+ * @internal
*/
_isMemoryOverloaded(sampleDurationMillis) {
const sample = this.snapshotter.getMemorySample(sampleDurationMillis);
@@ -171,6 +175,8 @@ class SystemStatus {
* @param {number} sampleDurationMillis
* @return {object}
* @ignore
+ * @protected
+ * @internal
*/
_isEventLoopOverloaded(sampleDurationMillis) {
const sample = this.snapshotter.getEventLoopSample(sampleDurationMillis);
@@ -184,6 +190,8 @@ class SystemStatus {
* @param {number} sampleDurationMillis
* @return {object}
* @ignore
+ * @protected
+ * @internal
*/
_isCpuOverloaded(sampleDurationMillis) {
const sample = this.snapshotter.getCpuSample(sampleDurationMillis);
@@ -195,7 +203,9 @@ class SystemStatus {
* if the client has been overloaded in the last sampleDurationMillis.
* @param {number} sampleDurationMillis
* @return {{isOverloaded: boolean, maxOverloadedRatio: number, actualRatio: number}}
- * @private
+ * @ignore
+ * @protected
+ * @internal
*/
_isClientOverloaded(sampleDurationMillis) {
const sample = this.snapshotter.getClientSample(sampleDurationMillis);
@@ -210,6 +220,8 @@ class SystemStatus {
* @param {number} ratio
* @return {object}
* @ignore
+ * @protected
+ * @internal
*/
_isSampleOverloaded(sample, ratio) { // eslint-disable-line class-methods-use-this
const weights = [];
diff --git a/src/browser_launchers/browser_launcher.js b/src/browser_launchers/browser_launcher.js
index bdd5ae766..b5a096876 100644
--- a/src/browser_launchers/browser_launcher.js
+++ b/src/browser_launchers/browser_launcher.js
@@ -1,10 +1,11 @@
import ow from 'ow';
import { ENV_VARS } from 'apify-shared/consts';
import { getTypicalChromeExecutablePath, isAtHome } from '../utils';
+import { BrowserPlugin } from './browser_plugin'; // eslint-disable-line no-unused-vars
/**
* @typedef BrowserLaunchContext
- * @property {object} [launchOptions]
+ * @property {Object<string, *>} [launchOptions]
* `Options passed to the browser launcher function. Options are based on underlying library.
* @property {string} [proxyUrl]
* URL to a HTTP proxy server. It must define the port number,
@@ -18,7 +19,7 @@ import { getTypicalChromeExecutablePath, isAtHome } from '../utils';
* is taken from the `APIFY_CHROME_EXECUTABLE_PATH` environment variable if provided,
* or defaults to the typical Google Chrome executable location specific for the operating system.
* By default, this option is `false`.
- * @property {Object} [launcher]
+ * @property {*} [launcher]
* By default this function uses require("playwright").chromium`.
* If you want to use a different browser you can pass it by this property as `require("playwright").firefox
*/
@@ -39,7 +40,7 @@ export default class BrowserLauncher {
*
* @param {string} launcher
* @param {string} apifyImageName
- * @returns {object}
+ * @returns {*}
*
*/
static requireLauncherOrThrow(launcher, apifyImageName) {
@@ -73,11 +74,15 @@ export default class BrowserLauncher {
this._validateProxyUrl(proxyUrl);
+ // those need to be reassigned otherwise they are {} in types
+ /** @type {*} */
this.launcher = launcher;
this.proxyUrl = proxyUrl;
this.useChrome = useChrome;
+ /** @type {Object<string, *>} */
this.launchOptions = launchOptions;
+ /** @type {BrowserPlugin} */
this.Plugin = null; // to be provided by child classes;
}
@@ -97,7 +102,7 @@ export default class BrowserLauncher {
/**
* Launches a browser instance based on the plugin.
- * @returns {object} Browser instance.
+ * @returns {Promise<*>} Browser instance.
*/
async launch() {
const plugin = this.createBrowserPlugin();
@@ -109,7 +114,7 @@ export default class BrowserLauncher {
}
/**
- * @returns {object}
+ * @returns {Object<string, *>}
*/
createLaunchOptions() {
const launchOptions = {
diff --git a/src/browser_launchers/browser_plugin.js b/src/browser_launchers/browser_plugin.js
new file mode 100644
index 000000000..a8b023693
--- /dev/null
+++ b/src/browser_launchers/browser_plugin.js
@@ -0,0 +1,22 @@
+// empty class to be properly typed
+export class BrowserPlugin {
+ /**
+ * @param {*} launcher
+ * @param {Object<string, *>} context
+ */
+ constructor(launcher, context) {
+ this.launcher = launcher;
+ this.context = context;
+ }
+
+ /** @return {Promise<*>} */
+ // eslint-disable-next-line no-empty-function
+ async createLaunchContext() { }
+
+ /**
+ * @param {Object<string, *>} context
+ * @return {Promise<*>}
+ */
+ // eslint-disable-next-line no-empty-function,no-unused-vars
+ async launch(context) { }
+}
diff --git a/src/browser_launchers/playwright_launcher.js b/src/browser_launchers/playwright_launcher.js
index 46d95f9c2..d1a044f5e 100644
--- a/src/browser_launchers/playwright_launcher.js
+++ b/src/browser_launchers/playwright_launcher.js
@@ -24,7 +24,7 @@ import BrowserLauncher from './browser_launcher';
* ```
*
* @typedef PlaywrightLaunchContext
- * @property {object} [launchOptions]
+ * @property {Object<string, *>} [launchOptions]
* `browserType.launch` [options](https://playwright.dev/docs/api/class-browsertype?_highlight=launch#browsertypelaunchoptions)
* @property {string} [proxyUrl]
* URL to a HTTP proxy server. It must define the port number,
@@ -38,7 +38,7 @@ import BrowserLauncher from './browser_launcher';
* is taken from the `APIFY_CHROME_EXECUTABLE_PATH` environment variable if provided,
* or defaults to the typical Google Chrome executable location specific for the operating system.
* By default, this option is `false`.
- * @property {Object} [launcher]
+ * @property {*} [launcher]
* By default this function uses `require("playwright").chromium`.
* If you want to use a different browser you can pass it by this property as e.g. `require("playwright").firefox`
*/
@@ -101,11 +101,11 @@ export class PlaywrightLauncher extends BrowserLauncher {
* for details.
*
*
- * @param {PlaywrightLaunchContext} [options]
+ * @param {PlaywrightLaunchContext} [launchContext]
* Optional settings passed to `browserType.launch()`. In addition to
* [Playwright's options](https://playwright.dev/docs/api/class-browsertype?_highlight=launch#browsertypelaunchoptions)
* the object may contain our own {@link PlaywrightLaunchContext} that enable additional features.
- * @returns {Promise<Browser>}
+ * @returns {Promise<*>}
* Promise that resolves to Playwright's `Browser` instance.
* @memberof module:Apify
* @name launchPlaywright
diff --git a/src/browser_launchers/puppeteer_launcher.js b/src/browser_launchers/puppeteer_launcher.js
index 119814a52..08d8ec745 100644
--- a/src/browser_launchers/puppeteer_launcher.js
+++ b/src/browser_launchers/puppeteer_launcher.js
@@ -32,7 +32,7 @@ const LAUNCH_PUPPETEER_DEFAULT_VIEWPORT = {
* ```
*
* @typedef PuppeteerLaunchContext
- * @property {object} [launchOptions]
+ * @property {Object<string, *>} [launchOptions]
* `puppeteer.launch` [options](https://pptr.dev/#?product=Puppeteer&version=v5.5.0&show=api-puppeteerlaunchoptions)
* @property {string} [proxyUrl]
* URL to a HTTP proxy server. It must define the port number,
@@ -50,7 +50,7 @@ const LAUNCH_PUPPETEER_DEFAULT_VIEWPORT = {
* is taken from the `APIFY_CHROME_EXECUTABLE_PATH` environment variable if provided,
* or defaults to the typical Google Chrome executable location specific for the operating system.
* By default, this option is `false`.
- * @property {Object} [launcher]
+ * @property {*} [launcher]
* Already required module (`Object`). This enables usage of various Puppeteer
* wrappers such as `puppeteer-extra`.
*
@@ -98,6 +98,7 @@ export class PuppeteerLauncher extends BrowserLauncher {
});
this.userAgent = userAgent;
this.stealth = stealth;
+ /** @type {StealthOptions} */
this.stealthOptions = {
hideWebDriver: true,
...stealthOptions,
@@ -186,7 +187,7 @@ export class PuppeteerLauncher extends BrowserLauncher {
* @param {PuppeteerLaunchContext} [launchContext]
* All `PuppeteerLauncher` parameters are passed via an launchContext object.
* If you want to pass custom `puppeteer.launch(options)` options you can use the `PuppeteerLaunchContext.launchOptions` property.
- * @returns {Promise<Browser>}
+ * @returns {Promise<*>}
* Promise that resolves to Puppeteer's `Browser` instance.
* @memberof module:Apify
* @name launchPuppeteer
diff --git a/src/cache_container.js b/src/cache_container.js
index e08e6cc02..733398ef2 100644
--- a/src/cache_container.js
+++ b/src/cache_container.js
@@ -5,7 +5,7 @@ import * as LruCache from 'apify-shared/lru_cache';
* or dataset cache. Before creation of this class, those caches were
* created as module scoped globals - untouchable. This proved problematic
* especially in tests, where caches would prevent test separation.
- * @property {Map<string, LruCache>} caches
+ * @property {Object<string, LruCache>} caches
* @ignore
*/
class CacheContainer {
diff --git a/src/crawlers/basic_crawler.js b/src/crawlers/basic_crawler.js
index 47ec83c65..e42477eca 100644
--- a/src/crawlers/basic_crawler.js
+++ b/src/crawlers/basic_crawler.js
@@ -13,15 +13,24 @@ import { validators } from '../validators';
// TYPE IMPORTS
/* eslint-disable no-unused-vars,import/named,import/no-duplicates,import/order */
import { AutoscaledPoolOptions } from '../autoscaling/autoscaled_pool';
+import { ProxyInfo } from '../proxy_configuration';
import Request from '../request';
import { RequestList } from '../request_list';
import { RequestQueue } from '../storages/request_queue';
import { QueueOperationInfo } from '../storages/request_queue';
import { Session } from '../session_pool/session';
import { SessionPoolOptions } from '../session_pool/session_pool';
-
/* eslint-enable no-unused-vars,import/named,import/no-duplicates,import/order */
+/**
+ * @typedef {object} CrawlingContext
+ * @property {string} id
+ * @property {Request} request
+ * @property {Session} session
+ * @property {ProxyInfo} proxyInfo
+ * @property {*} response
+ */
+
/**
* Since there's no set number of seconds before the container is terminated after
* a migration event, we need some reasonable number to use for RequestList persistence.
@@ -168,13 +177,13 @@ const SAFE_MIGRATION_WAIT_MILLIS = 20000;
* ```
* @property {Statistics} stats
* Contains statistics about the current run.
- * @property {?RequestList} requestList
+ * @property {RequestList} [requestList]
* A reference to the underlying {@link RequestList} class that manages the crawler's {@link Request}s.
* Only available if used by the crawler.
- * @property {?RequestQueue} requestQueue
+ * @property {RequestQueue} [requestQueue]
* A reference to the underlying {@link RequestQueue} class that manages the crawler's {@link Request}s.
* Only available if used by the crawler.
- * @property {?SessionPool} sessionPool
+ * @property {SessionPool} [sessionPool]
* A reference to the underlying {@link SessionPool} class that manages the crawler's {@link Session}s.
* Only available if used by the crawler.
* @property {AutoscaledPool} autoscaledPool
@@ -240,6 +249,8 @@ class BasicCrawler {
throw new ArgumentError(msg, this.constructor);
}
+ // assigning {} to the options as default break proper typing
+ /** @type {defaultLog.Log} */
this.log = log;
this.requestList = requestList;
this.requestQueue = requestQueue;
@@ -250,6 +261,7 @@ class BasicCrawler {
this.maxRequestRetries = maxRequestRetries;
this.handledRequestsCount = 0;
this.stats = new Statistics({ logMessage: `${log.getOptions().prefix} request statistics:` });
+ /** @type {SessionPoolOptions} */
this.sessionPoolOptions = {
...sessionPoolOptions,
log,
@@ -340,7 +352,9 @@ class BasicCrawler {
/**
* @return {Promise<void>}
- * @private
+ * @ignore
+ * @protected
+ * @internal
*/
async _init() {
// Initialize AutoscaledPool before awaiting _loadHandledRequestCount(),
@@ -356,14 +370,21 @@ class BasicCrawler {
}
/**
- * @param {object} crawlingContext
+ * @param {CrawlingContext} crawlingContext
* @return {Promise<void>}
- * @private
+ * @ignore
+ * @protected
+ * @internal
*/
async _handleRequestFunction(crawlingContext) { // eslint-disable-line no-unused-vars
await this.userProvidedHandler(crawlingContext);
}
+ /**
+ * @ignore
+ * @protected
+ * @internal
+ */
async _pauseOnMigration() {
if (this.autoscaledPool) {
// if run wasn't called, this is going to crash
@@ -406,6 +427,8 @@ class BasicCrawler {
* and RequestQueue is present then enqueues it to the queue first.
*
* @ignore
+ * @protected
+ * @internal
*/
async _fetchNextRequest() {
if (!this.requestList) return this.requestQueue.fetchNextRequest();
@@ -434,6 +457,8 @@ class BasicCrawler {
* then retries them in a case of an error, etc.
*
* @ignore
+ * @protected
+ * @internal
*/
async _runTaskFunction() {
const source = this.requestQueue || this.requestList;
@@ -496,6 +521,8 @@ class BasicCrawler {
* Returns true if either RequestList or RequestQueue have a request ready for processing.
*
* @ignore
+ * @protected
+ * @internal
*/
async _isTaskReadyFunction() {
// First check RequestList, since it's only in memory.
@@ -510,6 +537,8 @@ class BasicCrawler {
* Returns true if both RequestList and RequestQueue have all requests finished.
*
* @ignore
+ * @protected
+ * @internal
*/
async _defaultIsFinishedFunction() {
const [
@@ -531,6 +560,8 @@ class BasicCrawler {
* @param {(RequestList|RequestQueue)} source
* @return {Promise<void>}
* @ignore
+ * @protected
+ * @internal
*/
async _requestFunctionErrorHandler(error, crawlingContext, source) {
const { request } = crawlingContext;
@@ -562,7 +593,9 @@ class BasicCrawler {
* @param {Error} crawlingContext.error
* @param {Request} crawlingContext.request
* @return {Promise<void>}
- * @private
+ * @ignore
+ * @protected
+ * @internal
*/
async _handleFailedRequestFunction(crawlingContext) {
if (this.failedContextHandler) {
@@ -587,6 +620,8 @@ class BasicCrawler {
*
* @return {Promise<void>}
* @ignore
+ * @protected
+ * @internal
*/
async _loadHandledRequestCount() {
if (this.requestQueue) {
@@ -629,13 +664,13 @@ export default BasicCrawler;
/**
* @callback HandleFailedRequest
* @param {HandleFailedRequestInput} inputs Arguments passed to this callback.
- * @returns {(void|Promise<void>)}
+ * @returns {Promise<void>}
*/
/**
* @typedef HandleFailedRequestInput
* @property {Error} error The Error thrown by `handleRequestFunction`.
* @property {Request} request The original {Request} object.
- * @property {Session} [session]
- * @property {ProxyInfo} [proxyInfo]
+ * @property {Session} session
+ * @property {ProxyInfo} proxyInfo
*/
diff --git a/src/crawlers/browser_crawler.js b/src/crawlers/browser_crawler.js
index 2b52093a1..e560c1445 100644
--- a/src/crawlers/browser_crawler.js
+++ b/src/crawlers/browser_crawler.js
@@ -1,21 +1,53 @@
import ow from 'ow';
-import { BrowserPool } from 'browser-pool'; // eslint-disable-line import/no-duplicates
+import { BrowserPool, BrowserController } from 'browser-pool'; // eslint-disable-line import/no-duplicates,no-unused-vars
import { BASIC_CRAWLER_TIMEOUT_MULTIPLIER } from '../constants';
import { SessionPool } from '../session_pool/session_pool'; // eslint-disable-line import/no-duplicates
import EVENTS from '../session_pool/events'; // eslint-disable-line import/no-duplicates
import { addTimeoutToPromise } from '../utils';
-import BasicCrawler from './basic_crawler'; // eslint-disable-line import/no-duplicates
import { validators } from '../validators';
import {
throwOnBlockedRequest,
handleRequestTimeout,
} from './crawler_utils';
-// eslint-enable-line import/no-duplicates
+/* eslint-disable no-unused-vars,import/named,import/no-duplicates,import/order */
+import BasicCrawler, { CrawlingContext } from './basic_crawler';
+import { HandleFailedRequest } from './basic_crawler';
+import { ProxyConfiguration, ProxyInfo } from '../proxy_configuration';
+import { Session } from '../session_pool/session';
+import { BrowserPoolOptions } from 'browser-pool';
+import { RequestList } from '../request_list';
+import { RequestQueue } from '../storages/request_queue';
+import Request from '../request';
+import { SessionPoolOptions } from '../session_pool/session_pool';
+import { AutoscaledPoolOptions } from '../autoscaling/autoscaled_pool';
+/* eslint-enable no-unused-vars,import/named,import/no-duplicates,import/order */
+
+/**
+ * @typedef BrowserCrawlingContext
+ * @property {BrowserController} browserController
+ */
+/**
+ * @callback Hook
+ * @param {BrowserCrawlingContext & CrawlingContext} crawlingContext
+ * @param {Object<string,*>} gotoOptions
+ * @returns {Promise<void>}
+ */
+/**
+ * @callback BrowserHandlePageFunction
+ * @param {BrowserCrawlingContext & CrawlingContext} context
+ * @returns {Promise<void>}
+ */
+/**
+ * @callback GotoFunction
+ * @param {BrowserCrawlingContext & CrawlingContext} context
+ * @param {Object<string,*>} gotoOptions
+ * @returns {Promise<*>}
+ */
/**
* @typedef BrowserCrawlerOptions
- * @property {function} handlePageFunction
+ * @property {BrowserHandlePageFunction} handlePageFunction
* Function that is called to process each request.
* It is passed an object with the following fields:
*
@@ -55,7 +87,7 @@ import {
* {@link Request#pushErrorMessage} function.
* @property {number} [handlePageTimeoutSecs=60]
* Timeout in which the function passed as `handlePageFunction` needs to finish, in seconds.
- * @property {function} [gotoFunction]
+ * @property {GotoFunction} [gotoFunction]
* Navigation function for corresponding library. `page.goto(url)` is supported by both `playwright` and `puppeteer`.
* @property {HandleFailedRequest} [handleFailedRequestFunction]
* A function to handle requests that failed more than `option.maxRequestRetries` times.
@@ -84,7 +116,7 @@ import {
* If set, `PuppeteerCrawler` will be configured for all connections to use
* [Apify Proxy](https://my.apify.com/proxy) or your own Proxy URLs provided and rotated according to the configuration.
* For more information, see the [documentation](https://docs.apify.com/proxy).
- * @property {Array<function>} [preNavigationHooks]
+ * @property {Array<Hook>} [preNavigationHooks]
* Async functions that are sequentially evaluated before the navigation. Good for setting additional cookies
* or browser properties before navigation. The function accepts two parameters, `crawlingContext` and `gotoOptions`,
* which are passed to the `gotoFunction` the crawler calls to navigate.
@@ -92,11 +124,12 @@ import {
* ```
* preNavigationHooks: [
* async (crawlingContext, gotoOptions) => {
+ * const { page } = crawlingContext;
* await page.evaluate((attr) => { window.foo = attr; }, 'bar');
* }
* ]
* ```
- * @property {Array<function>} [postNavigationHooks]
+ * @property {Array<Hook>} [postNavigationHooks]
* Async functions that are sequentially evaluated after the navigation. Good for checking if the navigation was successful.
* The function accepts `crawlingContext` as an only parameter.
* Example:
@@ -198,16 +231,16 @@ import {
* ```
* @property {Statistics} stats
* Contains statistics about the current run.
- * @property {?RequestList} requestList
+ * @property {RequestList} [requestList]
* A reference to the underlying {@link RequestList} class that manages the crawler's {@link Request}s.
* Only available if used by the crawler.
- * @property {?RequestQueue} requestQueue
+ * @property {RequestQueue} [requestQueue]
* A reference to the underlying {@link RequestQueue} class that manages the crawler's {@link Request}s.
* Only available if used by the crawler.
- * @property {?SessionPool} sessionPool
+ * @property {SessionPool} [sessionPool]
* A reference to the underlying {@link SessionPool} class that manages the crawler's {@link Session}s.
* Only available if used by the crawler.
- * @property {?ProxyConfiguration} proxyConfiguration
+ * @property {ProxyConfiguration} [proxyConfiguration]
* A reference to the underlying {@link ProxyConfiguration} class that manages the crawler's proxies.
* Only available if used by the crawler.
* @property {BrowserPool} browserPool
@@ -219,7 +252,7 @@ import {
* You can use it to change the concurrency settings on the fly,
* to pause the crawler by calling {@link AutoscaledPool#pause}
* or to abort it by calling {@link AutoscaledPool#abort}.
- * @ignore
+ * @ignore
*/
class BrowserCrawler extends BasicCrawler {
static optionsShape = {
@@ -282,7 +315,9 @@ class BrowserCrawler extends BasicCrawler {
this.proxyConfiguration = proxyConfiguration;
+ /** @type {Array<Hook>} */
this.preNavigationHooks = preNavigationHooks;
+ /** @type {Array<Hook>} */
this.postNavigationHooks = postNavigationHooks;
if (useSessionPool) {
@@ -316,11 +351,10 @@ class BrowserCrawler extends BasicCrawler {
/**
* Wrapper around handlePageFunction that opens and closes pages etc.
*
- * @param {Object} crawlingContext
- * @param {Request} crawlingContext.request
- * @param {BrowserController} crawlingContext.browserController
- * @param {Session} [crawlingContext.session]
- * @private
+ * @param {BrowserCrawlingContext & CrawlingContext} crawlingContext
+ * @ignore
+ * @protected
+ * @internal
*/
async _handleRequestFunction(crawlingContext) {
const { id } = crawlingContext;
@@ -359,10 +393,11 @@ class BrowserCrawler extends BasicCrawler {
}
/**
- *
- * @param {object} crawlingContext
- * @param {Page} page
- * @private
+ * @param {BrowserCrawlingContext & CrawlingContext} crawlingContext
+ * @param {*} page
+ * @ignore
+ * @protected
+ * @internal
*/
_enhanceCrawlingContextWithPageInfo(crawlingContext, page) {
crawlingContext.page = page;
@@ -379,11 +414,13 @@ class BrowserCrawler extends BasicCrawler {
}
/**
- *
- * @param {object} crawlingContext
- * @private
+ * @param {BrowserCrawlingContext & CrawlingContext} crawlingContext
+ * @ignore
+ * @protected
+ * @internal
*/
async _handleNavigation(crawlingContext) {
+ /** @type {*} */
const gotoOptions = { ...this.defaultGotoOptions };
await this._executeHooks(this.preNavigationHooks, crawlingContext, gotoOptions);
try {
@@ -399,8 +436,11 @@ class BrowserCrawler extends BasicCrawler {
/**
* Marks session bad in case of navigation timeout.
- * @param {object} crawlingContext
+ * @param {BrowserCrawlingContext & CrawlingContext} crawlingContext
* @param {Error} error
+ * @ignore
+ * @protected
+ * @internal
*/
_handleNavigationTimeout(crawlingContext, error) {
const { session } = crawlingContext;
@@ -411,10 +451,11 @@ class BrowserCrawler extends BasicCrawler {
}
/**
- *
- * @param {object} crawlingContext
- * @param {object} gotoOptions
- * @private
+ * @param {BrowserCrawlingContext & CrawlingContext} crawlingContext
+ * @param {Object<string,*>} gotoOptions
+ * @ignore
+ * @protected
+ * @internal
*/
async _navigationHandler(crawlingContext, gotoOptions) {
if (!this.gotoFunction) {
@@ -428,9 +469,11 @@ class BrowserCrawler extends BasicCrawler {
/**
* Should be overriden in case of different automation library that does not support this response API.
* // @TODO: This can be also done as a postNavigation hook except the loadedUrl marking.
- * @param crawlingContext
+ * @param {BrowserCrawlingContext & CrawlingContext} crawlingContext
* @return {Promise<void>}
- * @private
+ * @ignore
+ * @protected
+ * @internal
*/
async _responseHandler(crawlingContext) {
const { response, session, request, page } = crawlingContext;
@@ -447,10 +490,11 @@ class BrowserCrawler extends BasicCrawler {
}
/**
- *
* @param {string} pageId
- * @param {object} launchContext
- * @private
+ * @param {*} launchContext
+ * @ignore
+ * @protected
+ * @internal
*/
async _extendLaunchContext(pageId, launchContext) {
const launchContextExtends = {};
@@ -460,7 +504,7 @@ class BrowserCrawler extends BasicCrawler {
}
if (this.proxyConfiguration) {
- const proxyInfo = await this.proxyConfiguration.newProxyInfo(launchContextExtends.session && launchContextExtends.session.id);
+ const proxyInfo = this.proxyConfiguration.newProxyInfo(launchContextExtends.session && launchContextExtends.session.id);
launchContext.proxyUrl = proxyInfo.url;
launchContextExtends.proxyInfo = proxyInfo;
}
@@ -469,11 +513,13 @@ class BrowserCrawler extends BasicCrawler {
}
/**
- *
- * @param {string} pageId
- * @param {BrowserController} browserController
- * @private
- */
+ *
+ * @param {string} pageId
+ * @param {BrowserController} browserController
+ * @ignore
+ * @protected
+ * @internal
+ */
_maybeAddSessionRetiredListener(pageId, browserController) {
if (this.sessionPool) {
const listener = (session) => {
@@ -489,10 +535,11 @@ class BrowserCrawler extends BasicCrawler {
}
/**
- *
- * @param {array} hooks
- * @param {...any} args
- * @private
+ * @param {Array<Hook>} hooks
+ * @param {*} args
+ * @ignore
+ * @protected
+ * @internal
*/
async _executeHooks(hooks, ...args) {
if (Array.isArray(hooks) && hooks.length) {
diff --git a/src/crawlers/cheerio_crawler.js b/src/crawlers/cheerio_crawler.js
index a72bdc712..996d033ad 100644
--- a/src/crawlers/cheerio_crawler.js
+++ b/src/crawlers/cheerio_crawler.js
@@ -16,8 +16,9 @@ import CrawlerExtension from './crawler_extension';
// TYPE IMPORTS
/* eslint-disable no-unused-vars,import/named,import/no-duplicates,import/order */
import { IncomingMessage } from 'http';
+import { Readable } from 'stream';
import AutoscaledPool, { AutoscaledPoolOptions } from '../autoscaling/autoscaled_pool';
-import { HandleFailedRequest } from './basic_crawler';
+import { HandleFailedRequest, CrawlingContext } from './basic_crawler';
import Request from '../request';
import { RequestList } from '../request_list';
import { ProxyConfiguration, ProxyInfo } from '../proxy_configuration';
@@ -460,10 +461,10 @@ class CheerioCrawler extends BasicCrawler {
/**
* Wrapper around handlePageFunction that opens and closes pages etc.
*
- * @param {Object} crawlingContext
- * @param {Request} crawlingContext.request
- * @param {Session} [crawlingContext.session]
+ * @param {CrawlingContext} crawlingContext
* @ignore
+ * @protected
+ * @internal
*/
async _handleRequestFunction(crawlingContext) {
const { request, session } = crawlingContext;
@@ -533,12 +534,14 @@ class CheerioCrawler extends BasicCrawler {
* on the request such as only downloading the request body if the
* received content type matches text/html, application/xml, application/xhtml+xml.
*
- * @param {Object} options
+ * @param {object} options
* @param {Request} options.request
* @param {Session} options.session
* @param {string} options.proxyUrl
* @returns {Promise<IncomingMessage|Readable>}
* @ignore
+ * @protected
+ * @internal
*/
async _requestFunction({ request, session, proxyUrl }) {
// Using the streaming API of Request to be able to
@@ -571,6 +574,8 @@ class CheerioCrawler extends BasicCrawler {
* @param {IncomingMessage|Readable} responseStream
* @returns {Promise<object>}
* @ignore
+ * @protected
+ * @internal
*/
async _parseResponse(request, responseStream) {
const { statusCode } = responseStream;
@@ -607,6 +612,8 @@ class CheerioCrawler extends BasicCrawler {
* @param {Session} [session]
* @param {string} [proxyUrl]
* @ignore
+ * @protected
+ * @internal
*/
_getRequestOptions(request, session, proxyUrl) {
const mandatoryRequestOptions = {
@@ -642,6 +649,14 @@ class CheerioCrawler extends BasicCrawler {
return { ...this.requestOptions, ...mandatoryRequestOptions };
}
+ /**
+ * @param {*} request
+ * @param {*} response
+ * @param {*} encoding
+ * @ignore
+ * @protected
+ * @internal
+ */
_encodeResponse(request, response, encoding) {
if (this.forceResponseEncoding) {
encoding = this.forceResponseEncoding;
@@ -675,6 +690,12 @@ class CheerioCrawler extends BasicCrawler {
throw new Error(`Resource ${request.url} served with unsupported charset/encoding: ${encoding}`);
}
+ /**
+ * @param {*} response
+ * @ignore
+ * @protected
+ * @internal
+ */
async _parseHtmlToDom(response) {
return new Promise((resolve, reject) => {
const domHandler = new htmlparser.DomHandler((err, dom) => {
@@ -691,6 +712,8 @@ class CheerioCrawler extends BasicCrawler {
* Checks and extends supported mime types
* @param {Array<(string|Object)>} additionalMimeTypes
* @ignore
+ * @protected
+ * @internal
*/
_extendSupportedMimeTypes(additionalMimeTypes) {
additionalMimeTypes.forEach((mimeType) => {
@@ -707,7 +730,9 @@ class CheerioCrawler extends BasicCrawler {
* Handles blocked request
* @param {Session} session
* @param {number} statusCode
- * @private
+ * @ignore
+ * @protected
+ * @internal
*/
_throwOnBlockedRequest(session, statusCode) {
const isBlocked = session.retireOnBlockedStatusCodes(statusCode);
@@ -720,7 +745,9 @@ class CheerioCrawler extends BasicCrawler {
/**
* Handles timeout request
* @param {Session} session
- * @private
+ * @ignore
+ * @protected
+ * @internal
*/
_handleRequestTimeout(session) {
if (session) session.markBad();
@@ -750,7 +777,7 @@ export default CheerioCrawler;
/**
* @typedef PostResponseInputs
- * @property {IncomingMessage|Readable} response stream
+ * @property {(IncomingMessage|Readable)} response stream
* @property {Request} request
* Original instance fo the {Request} object. Must be modified in-place.
* @property {Session} [session]
@@ -758,7 +785,7 @@ export default CheerioCrawler;
* @property {ProxyInfo} [proxyInfo]
* An object with information about currently used proxy by the crawler
* and configured by the {@link ProxyConfiguration} class.
- * @property {CheerioCrawler} [crawler]
+ * @property {CheerioCrawler} crawler
*/
/**
@@ -769,11 +796,11 @@ export default CheerioCrawler;
/**
* @typedef CheerioHandlePageInputs
- * @property {cheerio.Selector} [$]
+ * @property {cheerio.Root} $
* The [Cheerio](https://cheerio.js.org/) object with parsed HTML.
* @property {(string|Buffer)} body
* The request body of the web page.
- * @property {*} [json]
+ * @property {*} json
* The parsed object from JSON string if the response contains the content type application/json.
* @property {Request} request
* The original {@link Request} object.
@@ -781,11 +808,11 @@ export default CheerioCrawler;
* Parsed `Content-Type header: { type, encoding }`.
* @property {IncomingMessage} response
* An instance of Node's http.IncomingMessage object,
- * @property {Session} [session]
- * @property {ProxyInfo} [proxyInfo]
+ * @property {Session} session
+ * @property {ProxyInfo} proxyInfo
* An object with information about currently used proxy by the crawler
* and configured by the {@link ProxyConfiguration} class.
- * @property {CheerioCrawler} [crawler]
+ * @property {CheerioCrawler} crawler
*/
/**
diff --git a/src/crawlers/crawler_utils.js b/src/crawlers/crawler_utils.js
index ac9ef5ad0..1a8407044 100644
--- a/src/crawlers/crawler_utils.js
+++ b/src/crawlers/crawler_utils.js
@@ -1,3 +1,8 @@
+// TYPE IMPORTS
+/* eslint-disable no-unused-vars,import/named,import/no-duplicates,import/order */
+import { Session } from '../session_pool/session';
+/* eslint-enable no-unused-vars,import/named,import/no-duplicates,import/order */
+
/**
* Handles timeout request
* @param {Session} session
diff --git a/src/crawlers/playwright_crawler.js b/src/crawlers/playwright_crawler.js
index 180bd6968..b999ba7c3 100644
--- a/src/crawlers/playwright_crawler.js
+++ b/src/crawlers/playwright_crawler.js
@@ -1,10 +1,56 @@
import ow from 'ow';
-import BrowserCrawler from './browser_crawler';
-import { PlaywrightLauncher } from '../browser_launchers/playwright_launcher';
+
+/* eslint-disable no-unused-vars,import/named,import/no-duplicates,import/order */
+import { Page } from 'playwright';
+import { BrowserPoolOptions, BrowserPool } from 'browser-pool';
+import { PlaywrightLauncher, PlaywrightLaunchContext } from '../browser_launchers/playwright_launcher';
+import BrowserCrawler, { BrowserCrawlingContext } from './browser_crawler';
+import { HandleFailedRequest, CrawlingContext } from './basic_crawler';
+import { ProxyConfiguration, ProxyInfo } from '../proxy_configuration';
+import { SessionPoolOptions } from '../session_pool/session_pool';
+import { RequestList } from '../request_list';
+import { RequestQueue } from '../storages/request_queue';
+import Statistics from './statistics';
+import Request from '../request';
+import AutoscaledPool, { AutoscaledPoolOptions } from '../autoscaling/autoscaled_pool';
+/* eslint-enable no-unused-vars,import/named,import/no-duplicates,import/order */
+
import { gotoExtended } from '../playwright_utils';
+
+/**
+ * @typedef PlaywrightGotoOptions
+ * @property {number} [timeout]
+ * Maximum operation time in milliseconds, defaults to 30 seconds, pass `0` to disable timeout.
+ * The default value can be changed by using the browserContext.setDefaultNavigationTimeout(timeout),
+ * browserContext.setDefaultTimeout(timeout), page.setDefaultNavigationTimeout(timeout) or page.setDefaultTimeout(timeout) methods.
+ * @property {"domcontentloaded"|"load"|"networkidle"} [waitUntil]
+ * When to consider operation succeeded, defaults to `load`. Events can be either:
+ * - `'domcontentloaded'` - consider operation to be finished when the `DOMContentLoaded` event is fired.
+ * - `'load'` - consider operation to be finished when the `load` event is fired.
+ * - `'networkidle'` - consider operation to be finished when there are no network connections for at least `500` ms.
+ * @property {string} [referer]
+ * Referer header value. If provided it will take preference over the referer header value set by page.setExtraHTTPHeaders(headers).
+ */
+/**
+ * @callback PlaywrightHook
+ * @param {{ page: Page, crawler: PlaywrightCrawler } & BrowserCrawlingContext & CrawlingContext} crawlingContext
+ * @param {PlaywrightGotoOptions} gotoOptions
+ * @returns {Promise<void>}
+ */
+/**
+ * @typedef PlaywrightHandlePageFunctionParam
+ * @property {Page} page
+ * @property {PlaywrightCrawler} crawler
+ */
+/**
+ * @callback PlaywrightHandlePageFunction
+ * @param {PlaywrightHandlePageFunctionParam & BrowserCrawlingContext & CrawlingContext} context
+ * @returns {Promise<void>}
+ */
+
/**
* @typedef PlaywrightCrawlerOptions
- * @property {function} handlePageFunction
+ * @property {PlaywrightHandlePageFunction} handlePageFunction
* Function that is called to process each request.
* It is passed an object with the following fields:
*
@@ -58,7 +104,7 @@ import { gotoExtended } from '../playwright_utils';
* ```
* Where the {@link Request} instance corresponds to the failed request, and the `Error` instance
* represents the last error thrown during processing of the request.
- * @property {Array<function>} [preNavigationHooks]
+ * @property {Array<PlaywrightHook>} [preNavigationHooks]
* Async functions that are sequentially evaluated before the navigation. Good for setting additional cookies
* or browser properties before navigation. The function accepts two parameters, `crawlingContext` and `gotoOptions`,
* which are passed to the `gotoFunction` the crawler calls to navigate.
@@ -70,7 +116,7 @@ import { gotoExtended } from '../playwright_utils';
* }
* ]
* ```
- * @property {Array<function>} [postNavigationHooks]
+ * @property {Array<PlaywrightHook>} [postNavigationHooks]
* Async functions that are sequentially evaluated after the navigation. Good for checking if the navigation was successful.
* The function accepts `crawlingContext` as an only parameter.
* Example:
@@ -191,16 +237,16 @@ import { gotoExtended } from '../playwright_utils';
* ```
* @property {Statistics} stats
* Contains statistics about the current run.
- * @property {?RequestList} requestList
+ * @property {RequestList} requestList
* A reference to the underlying {@link RequestList} class that manages the crawler's {@link Request}s.
* Only available if used by the crawler.
- * @property {?RequestQueue} requestQueue
+ * @property {RequestQueue} requestQueue
* A reference to the underlying {@link RequestQueue} class that manages the crawler's {@link Request}s.
* Only available if used by the crawler.
- * @property {?SessionPool} sessionPool
+ * @property {SessionPool} sessionPool
* A reference to the underlying {@link SessionPool} class that manages the crawler's {@link Session}s.
* Only available if used by the crawler.
- * @property {?ProxyConfiguration} proxyConfiguration
+ * @property {ProxyConfiguration} proxyConfiguration
* A reference to the underlying {@link ProxyConfiguration} class that manages the crawler's proxies.
* Only available if used by the crawler.
* @property {BrowserPool} browserPool
@@ -227,7 +273,7 @@ class PlaywrightCrawler extends BrowserCrawler {
* @param {PlaywrightCrawlerOptions} options
* All `PlaywrightCrawler` parameters are passed via an options object.
*/
- constructor(options = {}) {
+ constructor(options) {
ow(options, 'PlaywrightCrawlerOptions', ow.object.exactShape(PlaywrightCrawler.optionsShape));
const {
@@ -265,6 +311,13 @@ class PlaywrightCrawler extends BrowserCrawler {
};
}
+ /**
+ * @param {*} crawlingContext
+ * @param {*} gotoOptions
+ * @ignore
+ * @protected
+ * @internal
+ */
async _navigationHandler(crawlingContext, gotoOptions) {
if (this.gotoFunction) {
this.log.deprecated('PlaywrightCrawler.gotoFunction is deprecated. Use "preNavigationHooks" and "postNavigationHooks" instead.');
diff --git a/src/crawlers/puppeteer_crawler.js b/src/crawlers/puppeteer_crawler.js
index ea786931f..aee3dd68d 100644
--- a/src/crawlers/puppeteer_crawler.js
+++ b/src/crawlers/puppeteer_crawler.js
@@ -1,10 +1,38 @@
import ow from 'ow';
-import BrowserCrawler from './browser_crawler';
import { gotoExtended } from '../puppeteer_utils';
-import { PuppeteerLauncher } from '../browser_launchers/puppeteer_launcher';
import applyStealthToBrowser from '../stealth/stealth';
+/* eslint-disable no-unused-vars,import/named,import/no-duplicates,import/order */
+import { PuppeteerLauncher, PuppeteerLaunchContext } from '../browser_launchers/puppeteer_launcher';
+import { HandleFailedRequest } from './basic_crawler';
+import BrowserCrawler, { BrowserCrawlingContext } from './browser_crawler';
+import { ProxyConfiguration } from '../proxy_configuration';
+import { SessionPoolOptions } from '../session_pool/session_pool';
+import { CrawlingContext } from './basic_crawler';
+import { Page, DirectNavigationOptions } from 'puppeteer';
+import { RequestList } from '../request_list';
+import { RequestQueue } from '../storages/request_queue';
+import { AutoscaledPoolOptions } from '../autoscaling/autoscaled_pool';
+import { BrowserPoolOptions } from 'browser-pool';
+/* eslint-enable no-unused-vars,import/named,import/no-duplicates,import/order */
+
+/**
+ * @callback PuppeteerHook
+ * @param {{ page: Page, crawler: PuppeteerCrawler } & BrowserCrawlingContext & CrawlingContext} crawlingContext
+ * @param {DirectNavigationOptions} gotoOptions
+ * @returns {Promise<void>}
+ */
+/**
+ * @typedef PuppeteerHandlePageFunctionParam
+ * @property {Page} page
+ * @property {PuppeteerCrawler} crawler
+ */
+/**
+ * @callback PuppeteerHandlePage
+ * @param {CrawlingContext & BrowserCrawlingContext & { page: Page, crawler: PuppeteerCrawler }} context
+ * @returns {Promise<void>}
+ */
/**
* @typedef PuppeteerCrawlerOptions
* @property {PuppeteerHandlePage} handlePageFunction
@@ -76,7 +104,7 @@ import applyStealthToBrowser from '../stealth/stealth';
* If set, `PuppeteerCrawler` will be configured for all connections to use
* [Apify Proxy](https://my.apify.com/proxy) or your own Proxy URLs provided and rotated according to the configuration.
* For more information, see the [documentation](https://docs.apify.com/proxy).
- * @property {Array<function>} [preNavigationHooks]
+ * @property {Array<PuppeteerHook>} [preNavigationHooks]
* Async functions that are sequentially evaluated before the navigation. Good for setting additional cookies
* or browser properties before navigation. The function accepts two parameters, `crawlingContext` and `gotoOptions`,
* which are passed to the `gotoFunction` the crawler calls to navigate.
@@ -84,11 +112,12 @@ import applyStealthToBrowser from '../stealth/stealth';
* ```
* preNavigationHooks: [
* async (crawlingContext, gotoOptions) => {
+ * const { page } = crawlingContext;
* await page.evaluate((attr) => { window.foo = attr; }, 'bar');
* }
* ]
* ```
- * @property {Array<function>} [postNavigationHooks]
+ * @property {Array<PuppeteerHook>} [postNavigationHooks]
* Async functions that are sequentially evaluated after the navigation. Good for checking if the navigation was successful.
* The function accepts `crawlingContext` as an only parameter.
* Example:
@@ -195,16 +224,16 @@ import applyStealthToBrowser from '../stealth/stealth';
* ```
* @property {Statistics} stats
* Contains statistics about the current run.
- * @property {?RequestList} requestList
+ * @property {RequestList} [requestList]
* A reference to the underlying {@link RequestList} class that manages the crawler's {@link Request}s.
* Only available if used by the crawler.
- * @property {?RequestQueue} requestQueue
+ * @property {RequestQueue} [requestQueue]
* A reference to the underlying {@link RequestQueue} class that manages the crawler's {@link Request}s.
* Only available if used by the crawler.
- * @property {?SessionPool} sessionPool
+ * @property {SessionPool} [sessionPool]
* A reference to the underlying {@link SessionPool} class that manages the crawler's {@link Session}s.
* Only available if used by the crawler.
- * @property {?ProxyConfiguration} proxyConfiguration
+ * @property {ProxyConfiguration} proxyConfiguration
* A reference to the underlying {@link ProxyConfiguration} class that manages the crawler's proxies.
* Only available if used by the crawler.
* @property {BrowserPool} browserPool
@@ -224,7 +253,6 @@ class PuppeteerCrawler extends BrowserCrawler {
gotoTimeoutSecs: ow.optional.number,
navigationTimeoutSecs: ow.optional.number,
launchContext: ow.optional.object,
-
}
/**
@@ -288,6 +316,13 @@ class PuppeteerCrawler extends BrowserCrawler {
};
}
+ /**
+ * @param {*} crawlingContext
+ * @param {*} gotoOptions
+ * @ignore
+ * @protected
+ * @internal
+ */
async _navigationHandler(crawlingContext, gotoOptions) {
if (this.gotoFunction) {
this.log.deprecated('PuppeteerCrawlerOptions.gotoFunction is deprecated. Use "preNavigationHooks" and "postNavigationHooks" instead.');
diff --git a/src/crawlers/statistics.js b/src/crawlers/statistics.js
index f525e128a..66aabe50f 100644
--- a/src/crawlers/statistics.js
+++ b/src/crawlers/statistics.js
@@ -74,7 +74,7 @@ class Statistics {
/**
* @private
- * @type {Map<string | number, Job>}
+ * @type {Object<string|number, Job>}
*/
this.requestsInProgress = new Map();
@@ -214,8 +214,10 @@ class Statistics {
}
/**
- * @private
* @param {Job} job
+ * @ignore
+ * @protected
+ * @internal
*/
_saveRetryCountForJob(job) {
const retryCount = job.retryCount();
@@ -241,7 +243,9 @@ class Statistics {
/**
* Loads the current statistic from the key value store if any
- * @private
+ * @ignore
+ * @protected
+ * @internal
*/
async _maybeLoadStatistics() {
// this might be called before startCapturing was called without using await, should not crash
@@ -284,7 +288,9 @@ class Statistics {
}
/**
- * @private
+ * @ignore
+ * @protected
+ * @internal
*/
_teardown() {
// this can be called before a call to startCapturing happens (or in a 'finally' block)
@@ -300,7 +306,7 @@ class Statistics {
* Make this class serializable when called with `JSON.stringify(statsInstance)` directly
* or through `keyValueStore.setValue('KEY', statsInstance)`
*
- * @returns {StatisticPersistedState | StatisticState}
+ * @returns {StatisticPersistedState & StatisticState}
*/
toJSON() {
// merge all the current state information that can be used from the outside
diff --git a/src/enqueue_links/click_elements.js b/src/enqueue_links/click_elements.js
index 455e72d06..decf85e7e 100644
--- a/src/enqueue_links/click_elements.js
+++ b/src/enqueue_links/click_elements.js
@@ -54,7 +54,7 @@ const STARTING_Z_INDEX = 2147400000;
* ],
* });
* ```
- * @param {Object} options
+ * @param {object} options
* All `enqueueLinksByClickingElements()` parameters are passed
* via an options object with the following keys:
* @param {Page} options.page
@@ -64,7 +64,7 @@ const STARTING_Z_INDEX = 2147400000;
* @param {string} options.selector
* A CSS selector matching elements to be clicked on. Unlike in {@link utils#enqueueLinks}, there is no default
* value. This is to prevent suboptimal use of this function by using it too broadly.
- * @param {Array<(string|RegExp|Object)>} [options.pseudoUrls]
+ * @param {Array<(string|RegExp|Object<string, *>)>} [options.pseudoUrls]
* An array of {@link PseudoUrl}s matching the URLs to be enqueued,
* or an array of strings or RegExps or plain Objects from which the {@link PseudoUrl}s can be constructed.
*
@@ -161,10 +161,12 @@ export async function enqueueLinksByClickingElements(options) {
* Catches and intercepts all initiated navigation requests and opened pages.
* Returns a list of all target URLs.
*
- * @param {Object} options
+ * @param {object} options
* @param {Page} options.page
* @param {string} options.selector
- * @return {Promise<Array<object>>}
+ * @param {number} [options.waitForPageIdleMillis]
+ * @param {number} [options.maxWaitForPageIdleMillis]
+ * @return {Promise<Array<*>>}
* @ignore
*/
export async function clickElementsAndInterceptNavigationRequests(options) {
diff --git a/src/enqueue_links/enqueue_links.js b/src/enqueue_links/enqueue_links.js
index 6af700525..3cba07778 100644
--- a/src/enqueue_links/enqueue_links.js
+++ b/src/enqueue_links/enqueue_links.js
@@ -7,7 +7,8 @@ import { constructPseudoUrlInstances, createRequests, addRequestsToQueueInBatche
// TYPE IMPORTS
/* eslint-disable no-unused-vars,import/named,import/no-duplicates,import/order */
-import { Page } from 'puppeteer';
+import { Page as PuppeteerPage } from 'puppeteer';
+import { Page as PlaywrightPage } from 'playwright';
import { RequestQueue, QueueOperationInfo } from '../storages/request_queue';
import { RequestTransform } from './shared';
import PseudoUrl from '../pseudo_url';
@@ -39,15 +40,15 @@ import { validators } from '../validators';
* });
* ```
*
- * @param {Object} options
+ * @param {object} options
* All `enqueueLinks()` parameters are passed
* via an options object with the following keys:
- * @param {Page} [options.page]
+ * @param {PuppeteerPage|PlaywrightPage} [options.page]
* Puppeteer [`Page`](https://pptr.dev/#?product=Puppeteer&show=api-class-page) object.
* Either `page` or `$` option must be provided.
* @param {Number} [options.limit]
* Limit the count of actually enqueued URLs to this number. Useful for testing across the entire crawling scope.
- * @param {CheerioStatic} [options.$]
+ * @param {(cheerio.Root|cheerio.Selector)} [options.$]
* [`Cheerio`](https://github.com/cheeriojs/cheerio) function with loaded HTML.
* Either `page` or `$` option must be provided.
* @param {RequestQueue} options.requestQueue
@@ -57,7 +58,7 @@ import { validators } from '../validators';
* @param {string} [options.baseUrl]
* A base URL that will be used to resolve relative URLs when using Cheerio. Ignored when using Puppeteer,
* since the relative URL resolution is done inside the browser automatically.
- * @param {Array<Object>|Array<string>} [options.pseudoUrls]
+ * @param {Array<Object<string, *>>|Array<string>} [options.pseudoUrls]
* An array of {@link PseudoUrl}s matching the URLs to be enqueued,
* or an array of strings or RegExps or plain Objects from which the {@link PseudoUrl}s can be constructed.
*
@@ -145,7 +146,7 @@ export async function enqueueLinks(options) {
/**
* Extracts URLs from a given Puppeteer Page.
*
- * @param {Page} page
+ * @param {PuppeteerPage|PlaywrightPage} page
* @param {string} selector
* @return {Promise<Array<string>>}
* @ignore
@@ -158,7 +159,7 @@ export async function extractUrlsFromPage(page, selector) {
/**
* Extracts URLs from a given Cheerio object.
*
- * @param {CheerioStatic} $
+ * @param {(cheerio.Root|cheerio.Selector)} $
* @param {string} selector
* @param {string} baseUrl
* @return {string[]}
diff --git a/src/events.js b/src/events.js
index ab9da2eb6..25c70377d 100644
--- a/src/events.js
+++ b/src/events.js
@@ -15,12 +15,14 @@ const events = new EventEmitter();
/**
* Websocket connection to actor events.
+ * @type {*}
* @ignore
*/
let eventsWs = null;
/**
* Interval that emits persist state events.
+ * @type {*}
* @ignore
*/
let persistStateInterval = null;
diff --git a/src/live_view/live_view_server.js b/src/live_view/live_view_server.js
index 3c3e6343f..ea3ca13a5 100644
--- a/src/live_view/live_view_server.js
+++ b/src/live_view/live_view_server.js
@@ -189,7 +189,9 @@ class LiveViewServer {
* Returns an absolute path to the screenshot with the given index.
* @param {number} screenshotIndex
* @return {string}
- * @private
+ * @ignore
+ * @protected
+ * @internal
*/
_getScreenshotPath(screenshotIndex) {
return path.join(this.screenshotDirectoryPath, `${screenshotIndex}.jpeg`);
@@ -198,7 +200,9 @@ class LiveViewServer {
/**
* @param {Page} page
* @return {Promise<Snapshot>}
- * @private
+ * @ignore
+ * @protected
+ * @internal
*/
async _makeSnapshot(page) {
const pageUrl = page.url();
@@ -225,7 +229,9 @@ class LiveViewServer {
/**
* @param {Snapshot} snapshot
- * @private
+ * @ignore
+ * @protected
+ * @internal
*/
_pushSnapshot(snapshot) {
// Send new snapshot to clients
@@ -236,13 +242,20 @@ class LiveViewServer {
/**
* Initiates an async delete and does not wait for it to complete.
* @param {number} screenshotIndex
- * @private
+ * @ignore
+ * @protected
+ * @internal
*/
_deleteScreenshot(screenshotIndex) {
unlink(this._getScreenshotPath(screenshotIndex))
.catch((err) => this.log.exception(err, 'Cannot delete live view screenshot.'));
}
+ /**
+ * @ignore
+ * @protected
+ * @internal
+ */
_setupHttpServer() {
const containerPort = process.env[ENV_VARS.CONTAINER_PORT] || LOCAL_ENV_VARS[ENV_VARS.CONTAINER_PORT];
@@ -278,7 +291,9 @@ class LiveViewServer {
/**
* @param {socketio.Socket} socket
- * @private
+ * @ignore
+ * @protected
+ * @internal
*/
_socketConnectionHandler(socket) {
this.clientCount++;
diff --git a/src/playwright_utils.js b/src/playwright_utils.js
index ae56deba2..cbc737793 100644
--- a/src/playwright_utils.js
+++ b/src/playwright_utils.js
@@ -1,8 +1,27 @@
import ow from 'ow';
-import _ from 'underscore';
-import { Page, Response, DirectNavigationOptions } from 'playwright'; // eslint-disable-line no-unused-vars
+import * as _ from 'underscore';
+import { Page, Response } from 'playwright'; // eslint-disable-line no-unused-vars
import log from './utils_log';
import { validators } from './validators';
+
+import Request from './request'; // eslint-disable-line import/named,no-unused-vars
+
+/**
+ * @typedef {object} DirectNavigationOptions
+ * @property {number} [timeout]
+ * Maximum operation time in milliseconds, defaults to 30 seconds, pass `0` to disable timeout. The
+ * default value can be changed by using the browserContext.setDefaultNavigationTimeout(timeout),
+ * browserContext.setDefaultTimeout(timeout), page.setDefaultNavigationTimeout(timeout) or
+ * page.setDefaultTimeout(timeout) methods.
+ * @property {("domcontentloaded"|"load"|"networkidle")} [waitUntil]
+ * When to consider operation succeeded, defaults to `load`. Events can be either:
+ * - `'domcontentloaded'` - consider operation to be finished when the `DOMContentLoaded` event is fired.
+ * - `'load'` - consider operation to be finished when the `load` event is fired.
+ * - `'networkidle'` - consider operation to be finished when there are no network connections for at least `500` ms.
+ * @property {string} [referer]
+ * Referer header value. If provided it will take preference over the referer header value set by page.setExtraHTTPHeaders(headers).
+ */
+
/**
* Extended version of Playwright's `page.goto()` allowing to perform requests with HTTP method other than GET,
* with custom headers and POST payload. URL, method, headers and payload are taken from
@@ -15,7 +34,7 @@ import { validators } from './validators';
* Puppeteer [`Page`](https://playwright.dev/docs/api/class-page) object.
* @param {Request} request
* @param {DirectNavigationOptions} [gotoOptions] Custom options for `page.goto()`.
- * @return {Promise<(Response | null)>}
+ * @return {Promise<(Response|null)>}
*
* @memberOf playwright
* @name gotoExtended
diff --git a/src/proxy_configuration.js b/src/proxy_configuration.js
index c46bd0bcb..a9db0e42c 100644
--- a/src/proxy_configuration.js
+++ b/src/proxy_configuration.js
@@ -13,6 +13,12 @@ const MAX_SESSION_ID_LENGTH = 50;
const CHECK_ACCESS_REQUEST_TIMEOUT_SECS = 4;
const CHECK_ACCESS_MAX_ATTEMPTS = 2;
+/**
+ * @callback ProxyConfigurationFunction
+ * @param {string|number} sessionId
+ * @returns {string}
+ */
+
/**
* @typedef ProxyConfigurationOptions
* @property {string} [password]
@@ -41,7 +47,7 @@ const CHECK_ACCESS_MAX_ATTEMPTS = 2;
* An array of custom proxy URLs to be rotated.
* Custom proxies are not compatible with Apify Proxy and an attempt to use both
* configuration options will cause an error to be thrown on initialize.
- * @property {function} [newUrlFunction]
+ * @property {ProxyConfigurationFunction} [newUrlFunction]
* Custom function that allows you to generate the new proxy URL dynamically. It gets the `sessionId` as a parameter
* and should always return stringified proxy URL.
* This function is used to generate the URL when {@link ProxyConfiguration.newUrl} or {@link ProxyConfiguration.newProxyInfo} is called.
@@ -277,6 +283,8 @@ export class ProxyConfiguration {
* @param {string} [sessionId]
* @return {string} the proxy username
* @ignore
+ * @protected
+ * @internal
*/
_getUsername(sessionId) {
let username;
@@ -314,6 +322,8 @@ export class ProxyConfiguration {
* and gets the password via API and sets it to env
* @returns {Promise<void>}
* @ignore
+ * @protected
+ * @internal
*/
async _setPasswordIfToken() {
const token = process.env[ENV_VARS.TOKEN] || LOCAL_ENV_VARS[ENV_VARS.TOKEN];
@@ -340,7 +350,9 @@ export class ProxyConfiguration {
* prevent program crashes caused by short downtimes of Proxy.
*
* @returns {Promise<void>}
+ * @protected
* @ignore
+ * @internal
*/
async _checkAccess() {
const status = await this._fetchStatus();
@@ -356,8 +368,10 @@ export class ProxyConfiguration {
/**
* Apify Proxy can be down for a second or a minute, but this should not crash processes.
*
- * @return {Promise<?{ connected: boolean, connectionError: string }>}
+ * @return {Promise<{ connected: boolean, connectionError: string }|undefined>}
+ * @protected
* @ignore
+ * @internal
*/
async _fetchStatus() {
const requestOpts = {
@@ -380,7 +394,9 @@ export class ProxyConfiguration {
* Handles custom url rotation with session
* @param {string} [sessionId]
* @returns {string} url
+ * @protected
* @ignore
+ * @internal
*/
_handleCustomUrl(sessionId) {
let customUrlToUse;
@@ -400,7 +416,9 @@ export class ProxyConfiguration {
/**
* Calls the custom newUrlFunction and checks format of its return value
* @param {string} [sessionId]
+ * @protected
* @ignore
+ * @internal
*/
_callNewUrlFunction(sessionId) {
let proxyUrl;
@@ -416,7 +434,9 @@ export class ProxyConfiguration {
/**
* Throws invalid custom newUrlFunction return
* @param {Error} err
+ * @protected
* @ignore
+ * @internal
*/
_throwNewUrlFunctionInvalid(err) {
throw new Error(`The provided newUrlFunction did not return a valid URL.\nCause: ${err.message}`);
@@ -426,7 +446,9 @@ export class ProxyConfiguration {
* Throws invalid proxy value error
* @param {string} param
* @param {string} value
+ * @protected
* @ignore
+ * @internal
*/
_throwInvalidProxyValueError(param, value) {
throw new Error(`The provided proxy ${param} name "${value}" can only contain the following characters: 0-9, a-z, A-Z, ".", "_" and "~"`);
@@ -434,7 +456,9 @@ export class ProxyConfiguration {
/**
* Throws Apify Proxy is not connected
+ * @protected
* @ignore
+ * @internal
*/
_throwApifyProxyConnectionError(errorMessage) {
throw new Error(errorMessage);
@@ -442,7 +466,9 @@ export class ProxyConfiguration {
/**
* Throws cannot combine custom proxies with Apify Proxy
+ * @protected
* @ignore
+ * @internal
*/
_throwCannotCombineCustomWithApify() {
throw new Error('Cannot combine custom proxies with Apify Proxy!'
@@ -452,7 +478,9 @@ export class ProxyConfiguration {
/**
* Throws cannot combine custom 2 custom methods
+ * @protected
* @ignore
+ * @internal
*/
_throwCannotCombineCustomMethods() {
throw new Error('Cannot combine custom proxies "options.proxyUrls" with custom generating function "options.newUrlFunction".');
@@ -494,12 +522,12 @@ export class ProxyConfiguration {
* { useApifyProxy: false }
* ```
*
-* @param {ProxyConfigurationOptions} [proxyConfigurationOptions]
-* @returns {Promise<?ProxyConfiguration>}
-* @memberof module:Apify
-* @name createProxyConfiguration
-* @function
- */
+ * @param {ProxyConfigurationOptions} [proxyConfigurationOptions]
+ * @returns {Promise<ProxyConfiguration|undefined>}
+ * @memberof module:Apify
+ * @name createProxyConfiguration
+ * @function
+ */
export const createProxyConfiguration = async (proxyConfigurationOptions = {}) => {
// Compatibility fix for Input UI where proxy: None returns { useApifyProxy: false }
// Without this, it would cause proxy to use the zero config / auto mode.
diff --git a/src/pseudo_url.js b/src/pseudo_url.js
index d0dff1a77..762030018 100644
--- a/src/pseudo_url.js
+++ b/src/pseudo_url.js
@@ -139,7 +139,7 @@ class PseudoUrl {
* be merged together, with the `userData` property having preference over the template.
* This enables dynamic overriding of the template.
*
- * @param {(string|Object)} urlOrProps
+ * @param {(string|Object<string, *>)} urlOrProps
* @return {Request}
*/
createRequest(urlOrProps) {
diff --git a/src/puppeteer_request_interception.js b/src/puppeteer_request_interception.js
index 00bc85532..785708b5f 100644
--- a/src/puppeteer_request_interception.js
+++ b/src/puppeteer_request_interception.js
@@ -43,8 +43,8 @@ class ObservableSet extends EventEmitter {
/**
* Makes all request headers capitalized to more look like in browser
- * @param {object} headers
- * @returns {object}
+ * @param {Object<string, string>} headers
+ * @returns {Object<string, string>}
*/
const browserifyHeaders = (headers) => {
const finalHeaders = {};
diff --git a/src/puppeteer_utils.js b/src/puppeteer_utils.js
index e32c3a2c9..bacb3e25d 100644
--- a/src/puppeteer_utils.js
+++ b/src/puppeteer_utils.js
@@ -19,6 +19,16 @@ const readFilePromised = util.promisify(fs.readFile);
const MAX_INJECT_FILE_CACHE_SIZE = 10;
const DEFAULT_BLOCK_REQUEST_URL_PATTERNS = ['.css', '.jpg', '.jpeg', '.png', '.svg', '.gif', '.woff', '.pdf', '.zip'];
+/**
+ * @typedef {object} CompiledScriptParams
+ * @property {Page} params.page
+ * @property {Request} params.request
+ */
+/**
+ * @callback CompiledScriptFunction
+ * @param {CompiledScriptParams} params
+ * @returns {Promise<*>}
+ */
/**
* Cache contents of previously injected files to limit file system access.
*/
@@ -34,7 +44,7 @@ const injectedFilesCache = new LruCache({ maxLength: MAX_INJECT_FILE_CACHE_SIZE
* @param {Page} page
* Puppeteer [`Page`](https://pptr.dev/#?product=Puppeteer&show=api-class-page) object.
* @param {string} filePath File path
- * @param {Object} [options]
+ * @param {object} [options]
* @param {boolean} [options.surviveNavigations]
* Enables the injected script to survive page navigations and reloads without need to be re-injected manually.
* This does not mean, however, that internal state will be preserved. Just that it will be automatically
@@ -160,7 +170,7 @@ const injectUnderscore = (page) => {
*
* @param {Page} page
* Puppeteer [`Page`](https://pptr.dev/#?product=Puppeteer&show=api-class-page) object.
- * @param {Object} [options]
+ * @param {object} [options]
* @param {string[]} [options.urlPatterns]
* The patterns of URLs to block from being loaded by the browser.
* Only `*` can be used as a wildcard. It is also automatically added to the beginning
@@ -212,7 +222,7 @@ const blockResources = async (page, resourceTypes = ['stylesheet', 'font', 'imag
* This issue should be resolved or atleast mitigated in future iterations of this feature.
* @param {Page} page
* Puppeteer [`Page`](https://pptr.dev/#?product=Puppeteer&show=api-class-page) object.
- * @param {Object} cache
+ * @param {Object<string, *>} cache
* Object in which responses are stored
* @param {Array<(string|RegExp)>} responseUrlRules
* List of rules that are used to check if the response should be cached.
@@ -293,8 +303,8 @@ const cacheResponses = async (page, cache, responseUrlRules) => {
* secured copies beforehand.
*
* @param {string} scriptString
- * @param {Object} context
- * @return {Function} `async ({ page, request }) => { scriptString }`
+ * @param {Object<string, *>} context
+ * @return {CompiledScriptFunction}
* @memberOf puppeteer
*/
const compileScript = (scriptString, context = Object.create(null)) => {
@@ -325,7 +335,7 @@ const compileScript = (scriptString, context = Object.create(null)) => {
* Puppeteer [`Page`](https://pptr.dev/#?product=Puppeteer&show=api-class-page) object.
* @param {Request} request
* @param {DirectNavigationOptions} [gotoOptions] Custom options for `page.goto()`.
- * @return {Promise<(Response | null)>}
+ * @return {Promise<(Response|null)>}
*
* @memberOf puppeteer
* @name gotoExtended
@@ -375,14 +385,14 @@ export const gotoExtended = async (page, request, gotoOptions = {}) => {
* Loads dynamic content when it hits the bottom of a page, and then continues scrolling.
* @param {Page} page
* Puppeteer [`Page`](https://pptr.dev/#?product=Puppeteer&show=api-class-page) object.
- * @param {Object} [options]
+ * @param {object} [options]
* @param {number} [options.timeoutSecs=0]
* How many seconds to scroll for. If 0, will scroll until bottom of page.
* @param {number} [options.waitForSecs=4]
* How many seconds to wait for no new content to load before exit.
- * @param {Boolean} [options.scrollDownAndUp=false]
+ * @param {boolean} [options.scrollDownAndUp=false]
* If true, it will scroll up a bit after each scroll down. This is required on some websites for the scroll to work.
- * @param {String} [options.buttonSelector]
+ * @param {string} [options.buttonSelector]
* Optionally checks and clicks a button if it appears while scrolling. This is required on some websites for the scroll to work.
* @returns {Promise<void>}
* @memberOf puppeteer
@@ -469,7 +479,7 @@ export const infiniteScroll = async (page, options = {}) => {
* Saves a full screenshot and HTML of the current page into a Key-Value store.
* @param {Page} page
* Puppeteer [`Page`](https://pptr.dev/#?product=Puppeteer&show=api-class-page) object.
- * @param {Object} [options]
+ * @param {object} [options]
* @param {string} [options.key=SNAPSHOT]
* Key under which the screenshot and HTML will be saved. `.jpg` will be appended for screenshot and `.html` for HTML.
* @param {number} [options.screenshotQuality=50]
@@ -478,7 +488,7 @@ export const infiniteScroll = async (page, options = {}) => {
* If true, it will save a full screenshot of the current page as a record with `key` appended by `.jpg`.
* @param {boolean} [options.saveHtml=true]
* If true, it will save a full HTML of the current page as a record with `key` appended by `.html`.
- * @param {string} [options.keyValueStoreName=null]
+ * @param {string|null} [options.keyValueStoreName=null]
* Name or id of the Key-Value store where snapshot is saved. By default it is saved to default Key-Value store.
* @returns {Promise<void>}
* @memberOf puppeteer
diff --git a/src/request.js b/src/request.js
index fc749147c..c0fcd3f95 100644
--- a/src/request.js
+++ b/src/request.js
@@ -83,9 +83,9 @@ const requestOptionalPredicates = {
* Indicates the number of times the crawling of the request has been retried on error.
* @property {string[]} errorMessages
* An array of error messages from request processing.
- * @property {object} headers
+ * @property {Object<string, string>} headers
* Object with HTTP headers. Key is header name, value is the value.
- * @property {object} userData
+ * @property {Object<string, *>} userData
* Custom user data assigned to the request.
* @property {Date} handledAt
* Indicates the time when the request has been processed.
@@ -144,7 +144,11 @@ class Request {
this.noRetry = noRetry;
this.retryCount = retryCount;
this.errorMessages = [...errorMessages];
+ // @property are ignored when reassigning, needs to enforced set again,
+ // otherwise the type will be {}
+ /** @type {Object<string, string>} */
this.headers = { ...headers };
+ /** @type {Object<string, any>} */
this.userData = { ...userData };
// Requests received from API will have ISOString dates,
// but we want to have a Date instance.
@@ -164,7 +168,7 @@ class Request {
* as possible, since just throwing a bad type error makes any debugging rather difficult.
*
* @param {(Error|string)} errorOrMessage Error object or error message to be stored in the request.
- * @param {Object} [options]
+ * @param {object} [options]
* @param {boolean} [options.omitStack=false] Only push the error message without stack trace when true.
*/
pushErrorMessage(errorOrMessage, options = {}) {
@@ -201,7 +205,8 @@ class Request {
/**
* @ignore
- * @private
+ * @protected
+ * @internal
*/
_computeUniqueKey({ url, method, payload, keepUrlFragment, useExtendedUniqueKey }) {
const normalizedMethod = method.toUpperCase();
@@ -245,7 +250,7 @@ export default Request;
* @property {string} [method='GET']
* @property {(string|Buffer)} [payload]
* HTTP request payload, e.g. for POST requests.
- * @property {Object} [headers={}]
+ * @property {Object<string,string>} [headers]
* HTTP headers in the following format:
* ```
* {
@@ -253,7 +258,7 @@ export default Request;
* 'Content-Type': 'application/json'
* }
* ```
- * @property {object} [userData={}]
+ * @property {Object<string,*>} [userData]
* Custom user data assigned to the request. Use this to save any request related data to the
* request's scope, keeping them accessible on retries, failures etc.
* @property {boolean} [keepUrlFragment=false]
diff --git a/src/request_list.js b/src/request_list.js
index 144bf54c6..1d0c6d0c8 100644
--- a/src/request_list.js
+++ b/src/request_list.js
@@ -52,7 +52,7 @@ const CONTENT_TYPE_BINARY = 'application/octet-stream';
* { requestsFromUrl: 'https://docs.google.com/spreadsheets/d/1GA5sSQhQjB_REes8I5IKg31S-TuRcznWOPjcpNqtxmU/gviz/tq?tqx=out:csv' }
* ]
* ```
- * @property {Function} [sourcesFunction]
+ * @property {RequestListSourcesFunction} [sourcesFunction]
* A function that will be called to get the sources for the `RequestList`, but only if `RequestList`
* was not able to fetch their persisted version (see {@link RequestListOptions.persistRequestsKey}).
* It must return an `Array` of {@link Request} or {@link RequestOptions}.
@@ -242,6 +242,7 @@ export class RequestList {
// Array of all requests from all sources, in the order as they appeared in sources.
// All requests in the array have distinct uniqueKey!
+ /** @type {Array<Request>} */
this.requests = [];
// Index to the next item in requests array to fetch. All previous requests are either handled or in progress.
@@ -316,6 +317,8 @@ export class RequestList {
* to a Stream once apify-client supports streams.
* @param {Buffer} persistedRequests
* @ignore
+ * @protected
+ * @internal
*/
async _addPersistedRequests(persistedRequests) {
// We don't need the sources so we purge them to
@@ -339,6 +342,8 @@ export class RequestList {
* to reduce memory footprint with very large sources.
* @returns {Promise<void>}
* @ignore
+ * @protected
+ * @internal
*/
async _addRequestsFromSources() {
// We'll load all sources in sequence to ensure that they get loaded in the right order.
@@ -403,6 +408,8 @@ export class RequestList {
*
* @return {Promise<void>}
* @ignore
+ * @protected
+ * @internal
*/
async _persistRequests() {
const serializedRequests = await serializeArray(this.requests);
@@ -415,6 +422,8 @@ export class RequestList {
*
* @param {RequestListState} state
* @ignore
+ * @protected
+ * @internal
*/
_restoreState(state) {
// If there's no state it means we've not persisted any (yet).
@@ -477,6 +486,8 @@ export class RequestList {
*
* @return {Promise<Array<(RequestListState|null)>>}
* @ignore
+ * @protected
+ * @internal
*/
async _loadStateAndPersistedRequests() {
let state;
@@ -519,7 +530,7 @@ export class RequestList {
* would return `null`, otherwise it resolves to `false`.
* Note that even if the list is empty, there might be some pending requests currently being processed.
*
- * @returns {Promise<Boolean>}
+ * @returns {Promise<boolean>}
*/
async isEmpty() {
this._ensureIsInitialized();
@@ -530,7 +541,7 @@ export class RequestList {
/**
* Returns `true` if all requests were already handled and there are no more left.
*
- * @returns {Promise<Boolean>}
+ * @returns {Promise<boolean>}
*/
async isFinished() {
this._ensureIsInitialized();
@@ -609,6 +620,8 @@ export class RequestList {
* Adds all fetched requests from a URL from a remote resource.
*
* @ignore
+ * @protected
+ * @internal
*/
async _addFetchedRequests(source, fetchedRequests) {
const { requestsFromUrl, regex } = source;
@@ -631,9 +644,11 @@ export class RequestList {
/**
* Fetches URLs from requestsFromUrl and returns them in format of list of requests
- * @param source
+ * @param {*} source
* @return {Promise<Array<RequestOptions>>}
* @ignore
+ * @protected
+ * @internal
*/
async _fetchRequestsFromUrl(source) {
const sharedOpts = _.omit(source, 'requestsFromUrl', 'regex');
@@ -662,8 +677,10 @@ export class RequestList {
* If the `source` parameter is a string or plain object and not an instance
* of a `Request`, then the function creates a `Request` instance.
*
- * @param {string|Request|object} source
+ * @param {(string|Request|object)} source
* @ignore
+ * @protected
+ * @internal
*/
_addRequest(source) {
let request;
@@ -702,6 +719,8 @@ export class RequestList {
* Throws an error if uniqueKey is not a non-empty string.
*
* @ignore
+ * @protected
+ * @internal
*/
_ensureUniqueKeyValid(uniqueKey) { // eslint-disable-line class-methods-use-this
if (typeof uniqueKey !== 'string' || !uniqueKey) {
@@ -713,6 +732,8 @@ export class RequestList {
* Checks that request is not reclaimed and throws an error if so.
*
* @ignore
+ * @protected
+ * @internal
*/
_ensureInProgressAndNotReclaimed(uniqueKey) {
if (!this.inProgress[uniqueKey]) {
@@ -727,6 +748,8 @@ export class RequestList {
* Throws an error if request list wasn't initialized.
*
* @ignore
+ * @protected
+ * @internal
*/
_ensureIsInitialized() {
if (!this.isInitialized) {
@@ -853,3 +876,8 @@ export const openRequestList = async (listName, sources, options = {}) => {
* @property {Object<string,boolean>} inProgress
* An object mapping request keys to a boolean value respresenting whether they are being processed at the moment.
*/
+
+/**
+ * @callback RequestListSourcesFunction
+ * @return {Promise<Array<(RequestOptions|Request|string)>>}
+ */
diff --git a/src/serialization.js b/src/serialization.js
index 99f4c44f8..2a8a78fb3 100644
--- a/src/serialization.js
+++ b/src/serialization.js
@@ -16,6 +16,7 @@ const pipeline = util.promisify(stream.pipeline);
* fashion to save memory. It operates in batches to speed
* up the process.
* @ignore
+ * @internal
*/
class ArrayToJson extends stream.Readable {
constructor(data, options = {}) {
@@ -59,7 +60,7 @@ class ArrayToJson extends stream.Readable {
* This could be further improved by outputting a Stream, if and when
* apify-client supports streams.
*
- * @param {Array} data
+ * @param {Array<*>} data
* @returns {Promise<Buffer>}
* @ignore
*/
@@ -83,7 +84,7 @@ export const serializeArray = async (data) => {
* when apify-client supports streams.
*
* @param {Buffer} compressedData
- * @returns {Promise<Array>}
+ * @returns {Promise<Array<*>>}
* @ignore
*/
export const deserializeArray = async (compressedData) => {
diff --git a/src/session_pool/session.js b/src/session_pool/session.js
index 1bc2855b7..8ac714395 100644
--- a/src/session_pool/session.js
+++ b/src/session_pool/session.js
@@ -35,7 +35,7 @@ const DEFAULT_SESSION_MAX_AGE_SECS = 3000;
* @typedef SessionOptions
* @property {string} [id] - Id of session used for generating fingerprints. It is used as proxy session name.
* @property {number} [maxAgeSecs=3000] - Number of seconds after which the session is considered as expired.
- * @property {object} [userData] - Object where custom user data can be stored. For example custom headers.
+ * @property {Object<string,*>} [userData] - Object where custom user data can be stored. For example custom headers.
* @property {number} [maxErrorScore=3] - Maximum number of marking session as blocked usage.
* If the `errorScore` reaches the `maxErrorScore` session is marked as block and it is thrown away.
* It starts at 0. Calling the `markBad` function increases the `errorScore` by 1.
@@ -107,6 +107,7 @@ export class Session {
this.cookieJar = cookieJar.setCookie ? cookieJar : CookieJar.fromJSON(JSON.stringify(cookieJar));
this.id = id;
this.maxAgeSecs = maxAgeSecs;
+ /** @type {Object<string,*>} */
this.userData = userData;
this.maxErrorScore = maxErrorScore;
this.errorScoreDecrement = errorScoreDecrement;
@@ -302,7 +303,9 @@ export class Session {
* Transforms puppeteer cookie to tough-cookie.
* @param {PuppeteerCookie} puppeteerCookie Cookie from puppeteer `page.cookies method.
* @return {Cookie}
- * @private
+ * @ignore
+ * @protected
+ * @internal
*/
_puppeteerCookieToTough(puppeteerCookie) {
const isExpiresValid = puppeteerCookie.expires && typeof puppeteerCookie.expires === 'number' && puppeteerCookie.expires > 0;
@@ -325,7 +328,9 @@ export class Session {
* Transforms tough-cookie to puppeteer cookie .
* @param {Cookie} toughCookie - Cookie from CookieJar
* @return {PuppeteerCookie} - Cookie from Puppeteer
- * @private
+ * @ignore
+ * @protected
+ * @internal
*/
_toughCookieToPuppeteer(toughCookie) {
return {
@@ -344,7 +349,9 @@ export class Session {
* Sets cookies.
* @param {Cookie[]} cookies
* @param {string} url
- * @private
+ * @ignore
+ * @protected
+ * @internal
*/
_setCookies(cookies, url) {
const errorMessages = [];
@@ -363,8 +370,11 @@ export class Session {
/**
* Calculate cookie expiration date
+ * @param {number} maxAgeSecs
* @return {Date} - calculated date by session max age seconds.
- * @private
+ * @ignore
+ * @protected
+ * @internal
*/
_getDefaultCookieExpirationDate(maxAgeSecs) {
return new Date(Date.now() + (maxAgeSecs * 1000));
@@ -372,7 +382,9 @@ export class Session {
/**
* Checks if session is not usable. if it is not retires the session.
- * @private
+ * @ignore
+ * @protected
+ * @internal
*/
_maybeSelfRetire() {
if (!this.isUsable()) {
diff --git a/src/session_pool/session_pool.js b/src/session_pool/session_pool.js
index 2b021d5f4..f2ec767c6 100644
--- a/src/session_pool/session_pool.js
+++ b/src/session_pool/session_pool.js
@@ -10,7 +10,7 @@ import { ACTOR_EVENT_NAMES_EX } from '../constants';
* Factory user-function which creates customized {@link Session} instances.
* @callback CreateSession
* @param {SessionPool} sessionPool Pool requesting the new session.
- * @returns {Session}
+ * @returns {Promise<Session>}
*/
/**
@@ -132,6 +132,7 @@ export class SessionPool extends EventEmitter {
// Operative states
this.keyValueStore = null;
+ /** @type {Session[]} */
this.sessions = [];
}
@@ -230,7 +231,9 @@ export class SessionPool extends EventEmitter {
/**
* Removes `Session` instance from `SessionPool`.
* @param {Session} session - Session to be removed
- * @private
+ * @ignore
+ * @protected
+ * @internal
*/
_removeSession(session) {
const sessionIndex = this.sessions.findIndex((storedSession) => storedSession.id === session.id);
@@ -242,7 +245,9 @@ export class SessionPool extends EventEmitter {
/**
* Adds `Session` instance to `SessionPool`.
* @param {Session} newSession `Session` instance to be added.
- * @private
+ * @ignore
+ * @protected
+ * @internal
*/
_addSession(newSession) {
this.sessions.push(newSession);
@@ -251,7 +256,9 @@ export class SessionPool extends EventEmitter {
/**
* Gets random index.
* @return {number}
- * @private
+ * @ignore
+ * @protected
+ * @internal
*/
_getRandomIndex() {
return Math.floor(Math.random() * this.sessions.length);
@@ -261,7 +268,9 @@ export class SessionPool extends EventEmitter {
* Creates new session without any extra behavior.
* @param {SessionPool} sessionPool
* @return {Session} - New session.
- * @private
+ * @ignore
+ * @protected
+ * @internal
*/
_defaultCreateSessionFunction(sessionPool) {
return new Session({
@@ -273,7 +282,9 @@ export class SessionPool extends EventEmitter {
/**
* Creates new session and adds it to the pool.
* @return {Promise<Session>} - Newly created `Session` instance.
- * @private
+ * @ignore
+ * @protected
+ * @internal
*/
async _createSession() {
const newSession = await this.createSessionFunction(this);
@@ -287,7 +298,9 @@ export class SessionPool extends EventEmitter {
/**
* Decides whether there is enough space for creating new session.
* @return {boolean}
- * @private
+ * @ignore
+ * @protected
+ * @internal
*/
_hasSpaceForSession() {
return this.sessions.length < this.maxPoolSize;
@@ -296,7 +309,9 @@ export class SessionPool extends EventEmitter {
/**
* Picks random session from the `SessionPool`.
* @return {Session} - Picked `Session`
- * @private
+ * @ignore
+ * @protected
+ * @internal
*/
_pickSession() {
return this.sessions[this._getRandomIndex()]; // Or maybe we should let the developer to customize the picking algorithm
@@ -306,7 +321,9 @@ export class SessionPool extends EventEmitter {
* Potentially loads `SessionPool`.
* If the state was persisted it loads the `SessionPool` from the persisted state.
* @return {Promise<void>}
- * @private
+ * @ignore
+ * @protected
+ * @internal
*/
async _maybeLoadSessionPool() {
const loadedSessionPool = await this.keyValueStore.getValue(this.persistStateKey);
diff --git a/src/stealth/stealth.js b/src/stealth/stealth.js
index 903bea66e..6f0c67224 100644
--- a/src/stealth/stealth.js
+++ b/src/stealth/stealth.js
@@ -80,8 +80,8 @@ function generateEvaluationDebugMessage() {
}
/**
* Logs the stealth errors in browser to the node stdout.
- * @param page {Page} - puppeteer page instance
- * @param evaluationDebugMessage {string} - debug message
+ * @param {Page} page - puppeteer page instance
+ * @param {string} evaluationDebugMessage - debug message
*/
function addStealthDebugToPage(page, evaluationDebugMessage) {
let warningLogged = false;
@@ -108,7 +108,7 @@ function addStealthDebugToPage(page, evaluationDebugMessage) {
/**
* Applies stealth tricks to the puppeteer page
* @param {Page} page
- * @param evaluationDebugMessage {string} - debug message
+ * @param {string} evaluationDebugMessage - debug message
* @param {StealthOptions} options
* @returns {Promise<void>}
* @private
diff --git a/src/storages/dataset.js b/src/storages/dataset.js
index 9b6c1b2e3..1bd747179 100644
--- a/src/storages/dataset.js
+++ b/src/storages/dataset.js
@@ -4,6 +4,13 @@ import { MAX_PAYLOAD_SIZE_BYTES } from 'apify-shared/consts';
import StorageManager from './storage_manager';
import log from '../utils_log';
+/* eslint-disable no-unused-vars,import/named,import/no-duplicates,import/order */
+// @ts-ignore
+import * as ApifyClient from 'apify-client';
+// @ts-ignore
+import * as ApifyStorageLocal from '@apify/storage-local';
+/* eslint-enable no-unused-vars,import/named,import/no-duplicates,import/order */
+
export const DATASET_ITERATORS_DEFAULT_LIMIT = 10000;
export const LOCAL_FILENAME_DIGITS = 9;
const SAFETY_BUFFER_PERCENT = 0.01 / 100; // 0.01%
@@ -13,7 +20,7 @@ const SAFETY_BUFFER_PERCENT = 0.01 / 100; // 0.01%
* and validates its serialized size against limitBytes. Optionally accepts its index
* in an array to provide better error messages. Returns serialized object.
*
- * @param {Object} item
+ * @param {*} item
* @param {number} limitBytes
* @param {number} [index]
* @returns {string}
diff --git a/src/storages/key_value_store.js b/src/storages/key_value_store.js
index cd275f2b8..e17123df7 100644
--- a/src/storages/key_value_store.js
+++ b/src/storages/key_value_store.js
@@ -6,6 +6,17 @@ import { APIFY_API_BASE_URL } from '../constants';
import StorageManager from './storage_manager';
import log from '../utils_log';
+/* eslint-disable no-unused-vars,import/named,import/no-duplicates,import/order */
+// @ts-ignore
+import * as ApifyClient from 'apify-client';
+// @ts-ignore
+import * as ApifyStorageLocal from '@apify/storage-local';
+/* eslint-enable no-unused-vars,import/named,import/no-duplicates,import/order */
+
+/**
+ * @typedef {(Object<string, *>|null|Buffer|string)} KeyValueStoreValueTypes
+ */
+
/**
* Helper function to possibly stringify value if options.contentType is not set.
*
@@ -141,7 +152,7 @@ export class KeyValueStore {
* @param {string} key
* Unique key of the record. It can be at most 256 characters long and only consist
* of the following characters: `a`-`z`, `A`-`Z`, `0`-`9` and `!-_.'()`
- * @returns {Promise<(object|Buffer|string|null)>}
+ * @returns {Promise<KeyValueStoreValueTypes>}
* Returns a promise that resolves to an object, string
* or [`Buffer`](https://nodejs.org/api/buffer.html), depending
* on the MIME content type of the record.
@@ -191,13 +202,13 @@ export class KeyValueStore {
* @param {string} key
* Unique key of the record. It can be at most 256 characters long and only consist
* of the following characters: `a`-`z`, `A`-`Z`, `0`-`9` and `!-_.'()`
- * @param {(Object|string|Buffer|null)} value
+ * @param {KeyValueStoreValueTypes} value
* Record data, which can be one of the following values:
* - If `null`, the record in the key-value store is deleted.
* - If no `options.contentType` is specified, `value` can be any JavaScript object and it will be stringified to JSON.
* - If `options.contentType` is set, `value` is taken as is and it must be a `String` or [`Buffer`](https://nodejs.org/api/buffer.html).
* For any other value an error will be thrown.
- * @param {Object} [options]
+ * @param {object} [options]
* @param {string} [options.contentType]
* Specifies a custom MIME content type of the record.
* @returns {Promise<void>}
@@ -275,7 +286,7 @@ export class KeyValueStore {
* ```
*
* @param {KeyConsumer} iteratee A function that is called for every key in the key-value store.
- * @param {Object} [options] All `forEachKey()` parameters are passed
+ * @param {object} [options] All `forEachKey()` parameters are passed
* via an options object with the following keys:
* @param {string} [options.exclusiveStartKey] All keys up to this one (including) are skipped from the result.
* @return {Promise<void>}
@@ -351,7 +362,7 @@ export const openKeyValueStore = async (storeIdOrName, options = {}) => {
*
* @param {string} key
* Unique record key.
- * @returns {Promise<object|string|Buffer|null>}
+ * @returns {Promise<Object<string, *>|string|Buffer|null>}
* Returns a promise that resolves to an object, string
* or [`Buffer`](https://nodejs.org/api/buffer.html), depending
* on the MIME content type of the record, or `null`
@@ -388,13 +399,13 @@ export const getValue = async (key) => {
*
* @param {string} key
* Unique record key.
- * @param {object} value
+ * @param {*} value
* Record data, which can be one of the following values:
* - If `null`, the record in the key-value store is deleted.
* - If no `options.contentType` is specified, `value` can be any JavaScript object and it will be stringified to JSON.
* - If `options.contentType` is set, `value` is taken as is and it must be a `String` or [`Buffer`](https://nodejs.org/api/buffer.html).
* For any other value an error will be thrown.
- * @param {Object} [options]
+ * @param {object} [options]
* @param {string} [options.contentType]
* Specifies a custom MIME content type of the record.
* @return {Promise<void>}
@@ -430,7 +441,7 @@ export const setValue = async (key, value, options) => {
* For more information, see {@link Apify#openKeyValueStore}
* and {@link KeyValueStore#getValue}.
*
- * @returns {Promise<object|string|Buffer|null>}
+ * @returns {Promise<Object<string, *>|string|Buffer|null>}
* Returns a promise that resolves to an object, string
* or [`Buffer`](https://nodejs.org/api/buffer.html), depending
* on the MIME content type of the record, or `null`
@@ -448,7 +459,7 @@ export const getInput = async () => getValue(process.env[ENV_VARS.INPUT_KEY] ||
* Current {KeyValue} key being processed.
* @param {number} index
* Position of the current key in {@link KeyValueStore}.
- * @param {object} info
+ * @param {*} info
* Information about the current {@link KeyValueStore} entry.
* @param {number} info.size
* Size of the value associated with the current key in bytes.
diff --git a/src/storages/request_queue.js b/src/storages/request_queue.js
index cd60cfcab..7f5877aaf 100644
--- a/src/storages/request_queue.js
+++ b/src/storages/request_queue.js
@@ -4,11 +4,18 @@ import * as ListDictionary from 'apify-shared/list_dictionary';
import { REQUEST_QUEUE_HEAD_MAX_LIMIT } from 'apify-shared/consts';
import { cryptoRandomObjectId } from 'apify-shared/utilities';
import ow from 'ow';
-import Request, { RequestOptions } from '../request'; // eslint-disable-line import/named,no-unused-vars
import StorageManager from './storage_manager';
import { sleep } from '../utils';
import log from '../utils_log';
+/* eslint-disable no-unused-vars,import/named,import/no-duplicates,import/order */
+import Request, { RequestOptions } from '../request'; // eslint-disable-line import/named,no-unused-vars
+// @ts-ignore
+import * as ApifyClient from 'apify-client';
+// @ts-ignore
+import * as ApifyStorageLocal from '@apify/storage-local';
+/* eslint-enable no-unused-vars,import/named,import/no-duplicates,import/order */
+
const MAX_CACHED_REQUESTS = 1000 * 1000;
// When requesting queue head we always fetch requestsInProgressCount * QUERY_HEAD_BUFFER number of requests.
@@ -230,7 +237,7 @@ export class RequestQueue {
* Gets the request from the queue specified by ID.
*
* @param {string} id ID of the request.
- * @return {Promise<(Request | null)>} Returns the request object, or `null` if it was not found.
+ * @return {Promise<(Request|null)>} Returns the request object, or `null` if it was not found.
*/
async getRequest(id) {
ow(id, ow.string);
@@ -370,7 +377,7 @@ export class RequestQueue {
* For example, this lets you store the number of retries or error messages for the request.
*
* @param {Request} request
- * @param {Object} [options]
+ * @param {object} [options]
* @param {boolean} [options.forefront=false]
* If `true` then the request it placed to the beginning of the queue, so that it's returned
* in the next call to {@link RequestQueue#fetchNextRequest}.
@@ -450,6 +457,8 @@ export class RequestQueue {
* @param {string} queueOperationInfo.requestId
* @param {boolean} queueOperationInfo.wasAlreadyHandled
* @ignore
+ * @protected
+ * @internal
*/
_cacheRequest(cacheKey, queueOperationInfo) {
this.requestsCache.add(cacheKey, {
@@ -468,6 +477,8 @@ export class RequestQueue {
* @param {number} [iteration] Used when this function is called recursively to limit the recursion.
* @return {Promise<boolean>} Indicates if queue head is consistent (true) or inconsistent (false).
* @ignore
+ * @protected
+ * @internal
*/
async _ensureHeadIsNonEmpty(
ensureConsistency = false,
diff --git a/src/storages/storage_manager.js b/src/storages/storage_manager.js
index 44ba282f0..4656f576c 100644
--- a/src/storages/storage_manager.js
+++ b/src/storages/storage_manager.js
@@ -2,6 +2,13 @@ import { ENV_VARS, LOCAL_ENV_VARS } from 'apify-shared/consts';
import cacheContainer from '../cache_container';
import * as utils from '../utils';
+/* eslint-disable no-unused-vars,import/named,import/no-duplicates,import/order */
+// @ts-ignore
+import * as ApifyClient from 'apify-client';
+// @ts-ignore
+import * as ApifyStorageLocal from '@apify/storage-local';
+/* eslint-enable no-unused-vars,import/named,import/no-duplicates,import/order */
+
const DEFAULT_ID_ENV_VAR_NAMES = {
Dataset: ENV_VARS.DEFAULT_DATASET_ID,
KeyValueStore: ENV_VARS.DEFAULT_KEY_VALUE_STORE_ID,
@@ -11,6 +18,7 @@ const DEFAULT_ID_ENV_VAR_NAMES = {
const MAX_OPENED_STORAGES = 1000;
/**
+ * @template T
* StorageManager takes care of opening remote or local storages.
* @property {Function} StorageConstructor
* @property {string} name
@@ -19,7 +27,7 @@ const MAX_OPENED_STORAGES = 1000;
*/
class StorageManager {
/**
- * @param {Function} StorageConstructor
+ * @param {T} StorageConstructor
*/
constructor(StorageConstructor) {
this.StorageConstructor = StorageConstructor;
@@ -31,7 +39,7 @@ class StorageManager {
* @param {string} idOrName
* @param {object} [options]
* @param {boolean} [options.forceCloud]
- * @return {Promise<object>}
+ * @return {Promise<T>}
*/
async openStorage(idOrName, options = {}) {
if (!process.env[ENV_VARS.LOCAL_STORAGE_DIR] && !process.env[ENV_VARS.TOKEN]) {
@@ -85,6 +93,9 @@ class StorageManager {
* @param {string} idOrName
* @param {boolean} isLocal
* @return {string}
+ * @ignore
+ * @protected
+ * @internal
*/
_createCacheKey(idOrName, isLocal) {
return isLocal
@@ -98,6 +109,8 @@ class StorageManager {
* @param {string} storageConstructorName
* @param {ApifyClient|ApifyStorageLocal} apiClient
* @ignore
+ * @protected
+ * @internal
*/
async _getOrCreateStorage(storageIdOrName, storageConstructorName, apiClient) {
const {
@@ -116,7 +129,9 @@ class StorageManager {
/**
* @param {ApifyClient|ApifyStorageLocal} client
* @param {string} storageConstructorName
- * @return {{ createStorageClient: function, createStorageCollectionClient: function }}
+ * @ignore
+ * @protected
+ * @internal
*/
_getStorageClientFactories(client, storageConstructorName) {
// Dataset => dataset
@@ -134,6 +149,9 @@ class StorageManager {
* @param {string} storage.id
* @param {string} [storage.name]
* @param {boolean} [storage.isLocal]
+ * @ignore
+ * @protected
+ * @internal
*/
_addStorageToCache(storage) {
const idKey = this._createCacheKey(storage.id, storage.isLocal);
diff --git a/src/typedefs.js b/src/typedefs.js
index d6ad3a1e3..3b2574669 100644
--- a/src/typedefs.js
+++ b/src/typedefs.js
@@ -21,7 +21,7 @@
* Status of the run. For possible values, see
* [Run lifecycle](https://docs.apify.com/actor/run#lifecycle)
* in Apify actor documentation.
- * @property {object} meta
+ * @property {Object<string, string>} meta
* Actor run meta-data. For example:
* ```
* {
@@ -30,7 +30,7 @@
* "userAgent": "ApifyClient/0.2.13 (Linux; Node/v8.11.3)"
* }
* ```
- * @property {object} stats
+ * @property {Object<string, number>} stats
* An object containing various actor run statistics. For example:
* ```
* {
@@ -40,7 +40,7 @@
* }
* ```
* Beware that object fields might change in future releases.
- * @property {object} options
+ * @property {Object<string, *>} options
* Actor run options. For example:
* ```
* {
@@ -69,7 +69,7 @@
* For more details, see
* [Container web server](https://docs.apify.com/actor/run#container-web-server)
* in Apify actor documentation.
- * @property {(object|null|undefined)} output
+ * @property {(Object<string, *>|null|undefined)} output
* Contains output of the actor run. The value is `null` or `undefined` in case the actor is still running,
* or if you pass `false` to the `fetchOutput` option of {@link Apify#call}.
*
diff --git a/src/utils.js b/src/utils.js
index a22a8841b..19b717132 100644
--- a/src/utils.js
+++ b/src/utils.js
@@ -370,10 +370,10 @@ export const getTypicalChromeExecutablePath = () => {
* Wraps the provided Promise with another one that rejects with the given errorMessage
* after the given timeoutMillis, unless the original promise resolves or rejects earlier.
*
- * @param {Promise<object>} promise
+ * @param {Promise<*>} promise
* @param {number} timeoutMillis
* @param {string} errorMessage
- * @return {Promise<object>}
+ * @return {Promise<*>}
* @ignore
*/
export const addTimeoutToPromise = (promise, timeoutMillis, errorMessage) => {
@@ -428,7 +428,7 @@ export const sleep = (millis) => {
* Returns a promise that resolves to an array of urls parsed from the resource available at the provided url.
* Optionally, custom regular expression and encoding may be provided.
*
- * @param {Object} options
+ * @param {object} options
* @param {string} options.url URL to the file
* @param {string} [options.encoding='utf8'] The encoding of the file.
* @param {RegExp} [options.urlRegExp=URL_NO_COMMAS_REGEX]
@@ -453,7 +453,7 @@ const downloadListOfUrls = async (options) => {
/**
* Collects all URLs in an arbitrary string to an array, optionally using a custom regular expression.
- * @param {Object} options
+ * @param {object} options
* @param {string} options.string
* @param {RegExp} [options.urlRegExp=Apify.utils.URL_NO_COMMAS_REGEX]
* @returns {string[]}
@@ -497,7 +497,7 @@ const BLOCK_TAGS_REGEX = /^(p|h1|h2|h3|h4|h5|h6|ol|ul|li|pre|address|blockquote|
* const html = '<html><body>Some text</body></html>';
* const text = htmlToText(cheerio.load(html, { decodeEntities: true }));
* ```
- * @param {(string|CheerioStatic)} html HTML text or parsed HTML represented using a
+ * @param {(string|cheerio.Root)} html HTML text or parsed HTML represented using a
* [cheerio](https://www.npmjs.com/package/cheerio) function.
* @return {string} Plain text
* @memberOf utils
@@ -512,7 +512,7 @@ const htmlToText = (html) => {
// produces really text with a lot of HTML elements in it. Let's just deprecate this sort of usage,
// and make the parameter "htmlOrCheerioElement"
/**
- * @type {CheerioStatic}
+ * @type {cheerio.Root}
* @ignore
*/
const $ = typeof html === 'function' ? html : cheerio.load(html, { decodeEntities: true });
@@ -558,12 +558,12 @@ const htmlToText = (html) => {
* Creates a standardized debug info from request and response. This info is usually added to dataset under the hidden `#debug` field.
*
* @param {(Request|RequestOptions)} request [Apify.Request](https://sdk.apify.com/docs/api/request) object.
- * @param {(IncomingMessage|PuppeteerResponse)} [response]
+ * @param {(*|IncomingMessage|PuppeteerResponse)} [response]
* Puppeteer [`Response`](https://pptr.dev/#?product=Puppeteer&version=v1.11.0&show=api-class-response)
* or NodeJS [`http.IncomingMessage`](https://nodejs.org/api/http.html#http_class_http_serverresponse).
- * @param {Object} [additionalFields] Object containing additional fields to be added.
+ * @param {Object<string, *>} [additionalFields] Object containing additional fields to be added.
- * @return {object}
+ * @return {Object<string, *>}
*/
const createRequestDebugInfo = (request, response = {}, additionalFields = {}) => {
ow(request, ow.object);
diff --git a/src/utils_log.js b/src/utils_log.js
index 44696f3f8..8c7dfeac1 100644
--- a/src/utils_log.js
+++ b/src/utils_log.js
@@ -4,16 +4,16 @@
// instead of 'apify-shared/log'
/**
- * @typedef {Object} LoggerOptions
+ * @typedef {object} LoggerOptions
* @property {number} [level=4] Sets the log level to the given value, preventing messages from less important log levels
* from being printed to the console. Use in conjunction with the `log.LEVELS` constants.
* @property {number} [maxDepth=4] Max depth of data object that will be logged. Anything deeper than the limit will be stripped off.
* @property {number} [maxStringLength=2000] Max length of the string to be logged. Longer strings will be truncated.
* @property {string} [prefix] Prefix to be prepended the each logged line.
* @property {string} [suffix] Suffix that will be appended the each logged line.
- * @property {Object} [logger] Logger implementation to be used. Default one is log.LoggerText to log messages as easily readable
+ * @property {object} [logger] Logger implementation to be used. Default one is log.LoggerText to log messages as easily readable
* strings. Optionally you can use `log.LoggerJson` that formats each log line as a JSON.
- * @property {Object} [data] Additional data to be added to each log line.
+ * @property {object} [data] Additional data to be added to each log line.
*/
/**
@@ -144,7 +144,7 @@
* to the message.
* @name debug
* @param {string} message
- * @param {Object} [data]
+ * @param {object} [data]
* @method
* @memberOf log
*/
@@ -154,7 +154,7 @@
* unless the log level is changed. Data are stringified and appended to the message.
* @name info
* @param {string} message
- * @param {Object} [data]
+ * @param {object} [data]
* @method
* @memberOf log
*/
@@ -163,7 +163,7 @@
* Logs a `WARNING` level message. Data are stringified and appended to the message.
* @name warning
* @param {string} message
- * @param {Object} [data]
+ * @param {object} [data]
* @method
* @memberOf log
*/
@@ -173,7 +173,7 @@
* to an exception. For logging exceptions, use the `log.exception` method.
* @name error
* @param {string} message
- * @param {Object} [data]
+ * @param {object} [data]
* @method
* @memberOf log
*/
@@ -184,7 +184,7 @@
* @name exception
* @param {Error} exception
* @param {string} [message]
- * @param {Object} [data]
+ * @param {object} [data]
* @method
* @memberOf log
*/
diff --git a/src/utils_request.js b/src/utils_request.js
index eaeeaf061..64d692bc5 100644
--- a/src/utils_request.js
+++ b/src/utils_request.js
@@ -17,13 +17,17 @@ const DEFAULT_HTTP_REQUEST_OPTIONS = {
timeoutSecs: 30,
};
+/**
+ * @typedef {(IncomingMessage & Readable & { body: string })} RequestAsBrowserResult
+ */
+
/**
* @typedef RequestAsBrowserOptions
* @property {string} url
* URL of the target endpoint. Supports both HTTP and HTTPS schemes.
- * @property {string} [method=GET]
+ * @property {string} [method="GET"]
* HTTP method.
- * @property {Object} [headers]
+ * @property {Object<string, string>} [headers]
* Additional HTTP headers to add. It's only recommended to use this option,
* with headers that are typically added by websites, such as cookies. Overriding
* default browser headers will remove the masking this function provides.
@@ -101,7 +105,7 @@ const DEFAULT_HTTP_REQUEST_OPTIONS = {
*
* @param {RequestAsBrowserOptions} options All `requestAsBrowser` configuration options.
*
- * @return {Promise<(IncomingMessage|Readable)>} This will typically be a
+ * @return {Promise<RequestAsBrowserResult>} This will typically be a
* [Node.js HTTP response stream](https://nodejs.org/api/http.html#http_class_http_incomingmessage),
* however, if returned from the cache it will be a [response-like object](https://github.com/lukechilds/responselike) which behaves in the same way.
* @memberOf utils
diff --git a/src/utils_social.js b/src/utils_social.js
index 3a3f7283c..a2e106fa6 100644
--- a/src/utils_social.js
+++ b/src/utils_social.js
@@ -537,7 +537,7 @@ try {
* ```
*
* @param {string} html HTML text
- * @param {Object} data Optional object which will receive the `text` and `$` properties
+ * @param {*|null} [data] Optional object which will receive the `text` and `$` properties
* that contain text content of the HTML and `cheerio` object, respectively. This is an optimization
* so that the caller doesn't need to parse the HTML document again, if needed.
* @return {SocialHandles} An object with the social handles.
diff --git a/test/typescript/crawlers/cheerio_crawler.test.ts b/test/typescript/crawlers/cheerio_crawler.test.ts
index 6f431d25a..5a796f567 100644
--- a/test/typescript/crawlers/cheerio_crawler.test.ts
+++ b/test/typescript/crawlers/cheerio_crawler.test.ts
@@ -21,6 +21,9 @@ describe('CheerioCrawler TS', () => {
testInputs = {
$: cheerio.load(body),
body: body,
+ json: null as any,
+ proxyInfo: null as any,
+ session: null as any,
request: new Apify.Request({url: ' http://www.test1234.com'}),
contentType: {type: 'text/html', encoding: 'utf-8'},
response: null as any,
diff --git a/tsconfig.json b/tsconfig.json
index c8cdc13c5..e4ef78ad5 100644
--- a/tsconfig.json
+++ b/tsconfig.json
@@ -21,6 +21,7 @@
"emitDeclarationOnly": false,
"strict": true,
"noImplicitAny": true,
+ "stripInternal": true,
"noImplicitReturns": true,
"noImplicitThis": true,
"alwaysStrict": true,
diff --git a/types-apify/apify-client.d.ts b/types-apify/apify-client.d.ts
deleted file mode 100644
index 14d2f728e..000000000
--- a/types-apify/apify-client.d.ts
+++ /dev/null
@@ -1,6 +0,0 @@
-declare module 'apify-client' {
- class ApifyClient {
- constructor(opts: any);
- }
- export = ApifyClient;
-}
diff --git a/types-apify/apify-shared/consts.d.ts b/types-apify/apify-shared/consts.d.ts
index bd7216b6f..a440a6af8 100644
--- a/types-apify/apify-shared/consts.d.ts
+++ b/types-apify/apify-shared/consts.d.ts
@@ -20,6 +20,11 @@ declare module 'apify-shared/consts' {
DEFAULT_KEY_VALUE_STORE_ID: string;
ACTOR_EVENTS_WS_URL: string;
INPUT_KEY: string;
+ HEADLESS: string;
+ XVFB: string;
+ PROXY_PASSWORD: string;
+ DEFAULT_REQUEST_QUEUE_ID: string;
+ CHROME_EXECUTABLE_PATH: string;
};
export const KEY_VALUE_STORE_KEYS: {
INPUT: string;
diff --git a/website/package.json b/website/package.json
index 9b56fbb38..5d1962d9d 100644
--- a/website/package.json
+++ b/website/package.json
@@ -14,9 +14,10 @@
"camelcase": "^5.3.1",
"decamelize": "^4.0.0",
"docusaurus": "^1.13.0",
- "jsdoc-to-markdown": "^5.0.1",
- "prettier": "^1.18.2",
- "fs-extra": "^8.1.0"
+ "fs-extra": "^8.1.0",
+ "jsdoc-plugin-intersection": "^1.0.4",
+ "jsdoc-to-markdown": "^7.0.0",
+ "prettier": "^1.18.2"
},
"dependencies": {
"@apify/http-request": "^1.1.5",
diff --git a/website/tools/build_docs.js b/website/tools/build_docs.js
index c1a58a737..39127e448 100644
--- a/website/tools/build_docs.js
+++ b/website/tools/build_docs.js
@@ -37,6 +37,7 @@ const getRenderOptions = (template, data) => ({
'property-list-format': 'list',
'heading-depth': 1,
helper: [path.join(__dirname, 'helpers.js')],
+ plugins: ["jsdoc-plugin-intersection"],
partial: [
path.join(__dirname, 'partials', 'params-list.hbs'),
path.join(__dirname, 'partials', 'properties-list.hbs'),
@@ -165,7 +166,10 @@ const main = async () => {
const typeFilesOutputDir = path.join(__dirname, '..', '..', 'docs', 'typedefs');
/* get template data */
- let templateData = await jsdoc2md.getTemplateData({ files: sourceFiles });
+ let templateData = await jsdoc2md.getTemplateData({
+ files: sourceFiles,
+ configure: path.join(__dirname, 'conf.json'),
+ });
// handle examples
await getExamplesFromRepo();
diff --git a/website/tools/conf.json b/website/tools/conf.json
new file mode 100644
index 000000000..955f80a16
--- /dev/null
+++ b/website/tools/conf.json
@@ -0,0 +1,3 @@
+{
+ "plugins": ["jsdoc-plugin-intersection"]
+}