Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Rebuild types [skip ci]

Ondra Urban294b7ae62ce0
diff --git a/types/autoscaling/snapshotter.d.ts b/types/autoscaling/snapshotter.d.ts
index 80b08859b..66967ce08 100644
--- a/types/autoscaling/snapshotter.d.ts
+++ b/types/autoscaling/snapshotter.d.ts
@@ -148,12 +148,12 @@ declare class Snapshotter {
* Starts capturing snapshots at configured intervals.
* @return {Promise}
*/
- start(): Promise<any>;
+ async start(): Promise<any>;
/**
* Stops all resource capturing.
* @return {Promise}
*/
- stop(): Promise<any>;
+ async stop(): Promise<any>;
/**
* Returns a sample of latest memory snapshots, with the size of the sample defined
* by the sampleDurationMillis parameter. If omitted, it returns a full snapshot history.
@@ -197,7 +197,7 @@ declare class Snapshotter {
* @return {Promise}
* @ignore
*/
- _snapshotMemoryOnLocal(intervalCallback: Function): Promise<any>;
+ async _snapshotMemoryOnLocal(intervalCallback: Function): Promise<any>;
/**
* Checks for critical memory overload and logs it to the console.
* @ignore
@@ -232,7 +232,7 @@ declare class Snapshotter {
* @param intervalCallback
* @private
*/
- _snapshotClient(intervalCallback: any): void;
+ private _snapshotClient;
/**
* Removes snapshots that are older than the snapshotHistorySecs option
* from the array (destructively - in place).
@@ -245,5 +245,5 @@ declare class Snapshotter {
* Calculate max memory for platform or local usage.
* @ignore
*/
- _ensureCorrectMaxMemory(): Promise<void>;
+ async _ensureCorrectMaxMemory(): Promise<void>;
}
diff --git a/types/autoscaling/system_status.d.ts b/types/autoscaling/system_status.d.ts
index c7c99bc28..f61333845 100644
--- a/types/autoscaling/system_status.d.ts
+++ b/types/autoscaling/system_status.d.ts
@@ -186,11 +186,7 @@ declare class SystemStatus {
* @return {{isOverloaded, maxOverloadedRatio, actualRatio}}
* @private
*/
- _isClientOverloaded(sampleDurationMillis: any): {
- isOverloaded: any;
- maxOverloadedRatio: any;
- actualRatio: any;
- };
+ private _isClientOverloaded;
/**
* Returns an object with sample information and an isOverloaded property
* set to true if at least the ratio of snapshots in the sample are overloaded.
diff --git a/types/crawlers/basic_crawler.d.ts b/types/crawlers/basic_crawler.d.ts
index d13f8fe38..ca9214c80 100644
--- a/types/crawlers/basic_crawler.d.ts
+++ b/types/crawlers/basic_crawler.d.ts
@@ -273,36 +273,36 @@ declare class BasicCrawler {
*
* @return {Promise<void>}
*/
- run(): Promise<void>;
+ async run(): Promise<void>;
autoscaledPool: AutoscaledPool;
sessionPool: import("../session_pool/session_pool").SessionPool;
- _pauseOnMigration(): Promise<void>;
+ async _pauseOnMigration(): Promise<void>;
/**
* Fetches request from either RequestList or RequestQueue. If request comes from a RequestList
* and RequestQueue is present then enqueues it to the queue first.
*
* @ignore
*/
- _fetchNextRequest(): Promise<any>;
+ async _fetchNextRequest(): Promise<any>;
/**
* Wrapper around handleRequestFunction that fetches requests from RequestList/RequestQueue
* then retries them in a case of an error, etc.
*
* @ignore
*/
- _runTaskFunction(): Promise<void>;
+ async _runTaskFunction(): Promise<void>;
/**
* Returns true if either RequestList or RequestQueue have a request ready for processing.
*
* @ignore
*/
- _isTaskReadyFunction(): Promise<boolean>;
+ async _isTaskReadyFunction(): Promise<boolean>;
/**
* Returns true if both RequestList and RequestQueue have all requests finished.
*
* @ignore
*/
- _defaultIsFinishedFunction(): Promise<any>;
+ async _defaultIsFinishedFunction(): Promise<any>;
/**
* Handles errors thrown by user provided handleRequestFunction()
* @param {Error} error
@@ -311,7 +311,7 @@ declare class BasicCrawler {
* @return {Promise<Boolean>} willBeRetried
* @ignore
*/
- _requestFunctionErrorHandler(error: Error, request: Request, source: RequestList | RequestQueue): Promise<boolean>;
+ async _requestFunctionErrorHandler(error: Error, request: Request, source: RequestList | RequestQueue): Promise<boolean>;
/**
* Updates handledRequestsCount from possibly stored counts,
* usually after worker migration. Since one of the stores
@@ -323,7 +323,7 @@ declare class BasicCrawler {
* @return {Promise<void>}
* @ignore
*/
- _loadHandledRequestCount(): Promise<void>;
+ async _loadHandledRequestCount(): Promise<void>;
}
import { RequestList } from "../request_list";
import { RequestQueue } from "../request_queue";
diff --git a/types/crawlers/cheerio_crawler.d.ts b/types/crawlers/cheerio_crawler.d.ts
index d36da7942..963483881 100644
--- a/types/crawlers/cheerio_crawler.d.ts
+++ b/types/crawlers/cheerio_crawler.d.ts
@@ -543,7 +543,7 @@ declare class CheerioCrawler {
*
* @return {Promise}
*/
- run(): Promise<any>;
+ async run(): Promise<any>;
autoscaledPool: any;
/**
* Wrapper around handlePageFunction that opens and closes pages etc.
@@ -554,7 +554,7 @@ declare class CheerioCrawler {
* @param {Session} options.session
* @ignore
*/
- _handleRequestFunction({ request, autoscaledPool, session }: {
+ async _handleRequestFunction({ request, autoscaledPool, session }: {
request: Request;
autoscaledPool: AutoscaledPool;
session: Session;
@@ -569,7 +569,7 @@ declare class CheerioCrawler {
* @param {Session} options.session
* @ignore
*/
- _requestFunction({ request, session }: {
+ async _requestFunction({ request, session }: {
request: Request;
session: Session;
}): Promise<{
@@ -620,7 +620,7 @@ declare class CheerioCrawler {
response: any;
encoding: any;
};
- _parseHtmlToDom(response: any): Promise<any>;
+ async _parseHtmlToDom(response: any): Promise<any>;
/**
* Checks and extends supported mime types
* @param {Array<String|Object>} additionalMimeTypes
@@ -633,13 +633,13 @@ declare class CheerioCrawler {
* @param statusCode {Number}
* @private
*/
- _throwOnBlockedRequest(session: Session, statusCode: number): void;
+ private _throwOnBlockedRequest;
/**
* Handles timeout request
* @param session {Session}
* @private
*/
- _handleRequestTimeout(session: Session): void;
+ private _handleRequestTimeout;
/**
* @param {Object} options
* @param {Error} options.error
@@ -647,7 +647,7 @@ declare class CheerioCrawler {
* @return {Promise<void>}
* @ignore
*/
- _defaultHandleFailedRequestFunction({ error, request }: {
+ async _defaultHandleFailedRequestFunction({ error, request }: {
error: Error;
request: Request;
}): Promise<void>;
diff --git a/types/crawlers/puppeteer_crawler.d.ts b/types/crawlers/puppeteer_crawler.d.ts
index b29196959..cb16b668e 100644
--- a/types/crawlers/puppeteer_crawler.d.ts
+++ b/types/crawlers/puppeteer_crawler.d.ts
@@ -465,7 +465,7 @@ declare class PuppeteerCrawler {
*
* @return {Promise<void>}
*/
- run(): Promise<void>;
+ async run(): Promise<void>;
sessionPool: import("../session_pool/session_pool").SessionPool;
isRunningPromise: Promise<void>;
autoscaledPool: AutoscaledPool;
@@ -477,7 +477,7 @@ declare class PuppeteerCrawler {
* @param {AutoscaledPool} options.autoscaledPool
* @ignore
*/
- _handleRequestFunction({ request, autoscaledPool }: {
+ async _handleRequestFunction({ request, autoscaledPool }: {
request: Request;
autoscaledPool: AutoscaledPool;
}): Promise<void>;
@@ -488,7 +488,7 @@ declare class PuppeteerCrawler {
* @return {Promise<PuppeteerResponse>}
* @ignore
*/
- _defaultGotoFunction({ page, request }: {
+ async _defaultGotoFunction({ page, request }: {
page: PuppeteerPage;
request: Request;
}): Promise<PuppeteerResponse>;
@@ -499,7 +499,7 @@ declare class PuppeteerCrawler {
* @return {Promise<void>}
* @ignore
*/
- _defaultHandleFailedRequestFunction({ error, request }: {
+ async _defaultHandleFailedRequestFunction({ error, request }: {
error: Error;
request: Request;
}): Promise<void>;
diff --git a/types/dataset.d.ts b/types/dataset.d.ts
index b69bc321f..048e0ac40 100644
--- a/types/dataset.d.ts
+++ b/types/dataset.d.ts
@@ -86,7 +86,7 @@ export class Dataset {
* The objects must be serializable to JSON and the JSON representation of each object must be smaller than 9MB.
* @return {Promise<void>}
*/
- pushData(data: any): Promise<void>;
+ async pushData(data: any): Promise<void>;
/**
* Returns {DatasetContent} object holding the items in the dataset based on the provided parameters.
*
@@ -147,7 +147,7 @@ export class Dataset {
* the legacy Apify Crawler product and it's not recommended to use it in new integrations.
* @return {Promise<DatasetContent>}
*/
- getData(options?: {
+ async getData(options?: {
format?: string;
offset?: number;
limit?: number;
@@ -192,7 +192,7 @@ export class Dataset {
*
* @returns {Promise<Object>}
*/
- getInfo(): Promise<any>;
+ async getInfo(): Promise<any>;
/**
* Iterates over dataset items, yielding each in turn to an `iteratee` function.
* Each invocation of `iteratee` is called with two arguments: `(item, index)`.
@@ -217,7 +217,7 @@ export class Dataset {
* @param {Number} [index=0] Specifies the initial index number passed to the `iteratee` function.
* @return {Promise<void>}
*/
- forEach(iteratee: DatasetConsumer, options?: {
+ async forEach(iteratee: DatasetConsumer, options?: {
desc?: boolean;
fields?: any[];
unwind?: string;
@@ -274,9 +274,9 @@ export class Dataset {
*
* @return {Promise<void>}
*/
- drop(): Promise<void>;
+ async drop(): Promise<void>;
/** @ignore */
- delete(): Promise<void>;
+ async delete(): Promise<void>;
}
/**
* This is a local emulation of a dataset.
@@ -294,14 +294,14 @@ export class DatasetLocal {
initializationPromise: any;
_initialize(): any;
pushData(data: any): any;
- getData(opts?: {}): Promise<{
+ async getData(opts?: {}): Promise<{
items: any[];
total: number;
offset: any;
count: number;
limit: any;
}>;
- getInfo(): Promise<{
+ async getInfo(): Promise<{
id: any;
name: any;
userId: string;
@@ -311,11 +311,11 @@ export class DatasetLocal {
itemCount: number;
cleanItemCount: number;
}>;
- forEach(iteratee: any): Promise<void>;
- map(iteratee: any): Promise<any[]>;
- reduce(iteratee: any, memo: any): Promise<any>;
- drop(): Promise<void>;
- delete(): Promise<void>;
+ async forEach(iteratee: any): Promise<void>;
+ async map(iteratee: any): Promise<any[]>;
+ async reduce(iteratee: any, memo: any): Promise<any>;
+ async drop(): Promise<void>;
+ async delete(): Promise<void>;
/**
* Returns an array of item indexes for given offset and limit.
*/
diff --git a/types/key_value_store.d.ts b/types/key_value_store.d.ts
index b0c6832c7..08b3c316b 100644
--- a/types/key_value_store.d.ts
+++ b/types/key_value_store.d.ts
@@ -163,9 +163,9 @@ export class KeyValueStore {
*
* @return {Promise<void>}
*/
- drop(): Promise<void>;
+ async drop(): Promise<void>;
/** @ignore */
- delete(): Promise<void>;
+ async delete(): Promise<void>;
/**
* Returns a URL for the given key that may be used to publicly
* access the value in the remote key value store.
@@ -198,7 +198,7 @@ export class KeyValueStore {
* @param {string} [options.exclusiveStartKey] All keys up to this one (including) are skipped from the result.
* @return {Promise<void>}
*/
- forEachKey(iteratee: KeyConsumer, options?: {
+ async forEachKey(iteratee: KeyConsumer, options?: {
exclusiveStartKey?: string;
}, index?: number): Promise<void>;
}
@@ -213,11 +213,11 @@ export class KeyValueStoreLocal {
localStoragePath: string;
storeId: any;
initializationPromise: any;
- getValue(key: any): Promise<any>;
- setValue(key: any, value: any, options?: {}): Promise<void>;
- delete(): Promise<void>;
- drop(): Promise<void>;
- forEachKey(iteratee: any, options?: {}, index?: number): Promise<void>;
+ async getValue(key: any): Promise<any>;
+ async setValue(key: any, value: any, options?: {}): Promise<void>;
+ async delete(): Promise<void>;
+ async drop(): Promise<void>;
+ async forEachKey(iteratee: any, options?: {}, index?: number): Promise<void>;
/**
* Helper function to handle files. Accepts a promisified 'fs' function as a second parameter
* which will be executed against the file saved under the key. Since the file's extension and thus
@@ -235,7 +235,7 @@ export class KeyValueStoreLocal {
* }
* @ignore
*/
- _handleFile(key: string, handler: Function): Promise<any>;
+ async _handleFile(key: string, handler: Function): Promise<any>;
/**
* Performs a lookup for a file in the local emulation directory's file list.
* @param {String} key
diff --git a/types/live_view/live_view_server.d.ts b/types/live_view/live_view_server.d.ts
index 173855c64..14624a06b 100644
--- a/types/live_view/live_view_server.d.ts
+++ b/types/live_view/live_view_server.d.ts
@@ -66,7 +66,7 @@ declare class LiveViewServer {
* @type {?Snapshot}
* @private
*/
- lastSnapshot: Snapshot | null;
+ private lastSnapshot;
lastScreenshotIndex: number;
clientCount: number;
_isRunning: boolean;
@@ -78,13 +78,13 @@ declare class LiveViewServer {
* Snapshots will not be created until a client has connected.
* @return {Promise}
*/
- start(): Promise<any>;
+ async start(): Promise<any>;
/**
* Prevents the server from receiving more connections. Existing connections
* will not be terminated, but the server will not prevent a process exit.
* @return {Promise}
*/
- stop(): Promise<any>;
+ async stop(): Promise<any>;
/**
* Serves a snapshot to all connected clients.
* Screenshots are not served directly, only their index number
@@ -95,7 +95,7 @@ declare class LiveViewServer {
* @param {Page} page
* @return {Promise}
*/
- serve(page: Page): Promise<any>;
+ async serve(page: Page): Promise<any>;
/**
* @return {boolean}
*/
@@ -110,24 +110,24 @@ declare class LiveViewServer {
* @return {string}
* @private
*/
- _getScreenshotPath(screenshotIndex: number): string;
+ private _getScreenshotPath;
/**
* @param {Page} page
* @return {Promise<Snapshot>}
* @private
*/
- _makeSnapshot(page: Page): Promise<Snapshot>;
+ private async _makeSnapshot;
/**
* @param {Snapshot} snapshot
* @private
*/
- _pushSnapshot(snapshot: Snapshot): void;
+ private _pushSnapshot;
/**
* Initiates an async delete and does not wait for it to complete.
* @param {number} screenshotIndex
* @private
*/
- _deleteScreenshot(screenshotIndex: number): void;
+ private _deleteScreenshot;
_setupHttpServer(): void;
port: number;
liveViewUrl: any;
@@ -135,9 +135,8 @@ declare class LiveViewServer {
* @param {socketio.Socket} socket
* @private
*/
- _socketConnectionHandler(socket: socketio.Socket): void;
+ private _socketConnectionHandler;
}
-import Snapshot from "./snapshot";
import http from "http";
import socketio from "socket.io";
import { Page } from "puppeteer";
diff --git a/types/puppeteer_pool.d.ts b/types/puppeteer_pool.d.ts
index a59106b15..811e50910 100644
--- a/types/puppeteer_pool.d.ts
+++ b/types/puppeteer_pool.d.ts
@@ -192,7 +192,7 @@ declare class PuppeteerPool {
pagesToInstancesMap: WeakMap<object, any>;
liveViewSnapshotsInProgress: WeakMap<object, any>;
sigintListener: () => void;
- _retireBrowserWithSession(session: any): Promise<void>;
+ async _retireBrowserWithSession(session: any): Promise<void>;
/**
* Launches new browser instance.
*
@@ -206,7 +206,7 @@ declare class PuppeteerPool {
* @returns {Promise<void>}
* @ignore
*/
- _initBrowser(browserPromise: Promise<Browser>, instance: PuppeteerInstance): Promise<void>;
+ async _initBrowser(browserPromise: Promise<Browser>, instance: PuppeteerInstance): Promise<void>;
/**
* Retires some of the instances for example due to many uses.
*
@@ -226,7 +226,7 @@ declare class PuppeteerPool {
*
* @ignore
*/
- _killInstance(instance: any): Promise<void>;
+ async _killInstance(instance: any): Promise<void>;
/**
* Kills all running PuppeteerInstances.
* @ignore
@@ -248,7 +248,7 @@ declare class PuppeteerPool {
*
* @return {Promise<Page>}
*/
- newPage(): Promise<Page>;
+ async newPage(): Promise<Page>;
/**
* Opens new tab in one of the browsers in the pool and returns a `Promise`
* that resolves to an instance of a Puppeteer
@@ -257,7 +257,7 @@ declare class PuppeteerPool {
* @return {Promise<Page>}
* @ignore
*/
- _openNewTab(): Promise<Page>;
+ async _openNewTab(): Promise<Page>;
/**
* Adds the necessary boilerplate to allow page reuse and also
* captures page.close() errors to prevent meaningless log clutter.
@@ -272,19 +272,19 @@ declare class PuppeteerPool {
* @param {Browser} browser
* @ignore
*/
- _focusOldestTab(browser: Browser): Promise<void>;
+ async _focusOldestTab(browser: Browser): Promise<void>;
/**
* Closes all open browsers.
* @return {Promise<void>}
*/
- destroy(): Promise<void>;
+ async destroy(): Promise<void>;
/**
* Finds a PuppeteerInstance given a Puppeteer Browser running in the instance.
* @param {Browser} browser
* @return {Promise}
* @ignore
*/
- _findInstanceByBrowser(browser: Browser): Promise<any>;
+ async _findInstanceByBrowser(browser: Browser): Promise<any>;
/**
* Manually retires a Puppeteer
* [`Browser`](https://pptr.dev/#?product=Puppeteer&show=api-class-browser)
@@ -293,7 +293,7 @@ declare class PuppeteerPool {
* @param {Browser} browser
* @return {Promise<void>}
*/
- retire(browser: Browser): Promise<void>;
+ async retire(browser: Browser): Promise<void>;
/**
* Closes the page, unless the `reuseTabs` option is set to true.
* Then it would only flag the page for a future reuse, without actually closing it.
@@ -305,14 +305,14 @@ declare class PuppeteerPool {
* @param {Page} page
* @return {Promise<void>}
*/
- recyclePage(page: Page): Promise<void>;
+ async recyclePage(page: Page): Promise<void>;
/**
* Tells the connected LiveViewServer to serve a snapshot when available.
*
* @param page
* @return {Promise<void>}
*/
- serveLiveViewSnapshot(page: any): Promise<void>;
+ async serveLiveViewSnapshot(page: any): Promise<void>;
_findInstancesBySession(session: any): any[];
_killInstanceWithNoPages(instance: any): void;
}
diff --git a/types/request_list.d.ts b/types/request_list.d.ts
index 9c8a5792e..86296499b 100644
--- a/types/request_list.d.ts
+++ b/types/request_list.d.ts
@@ -164,7 +164,7 @@ export class RequestList {
*
* @returns {Promise<void>}
*/
- initialize(): Promise<void>;
+ async initialize(): Promise<void>;
/**
* Persists the current state of the `RequestList` into the default {@link KeyValueStore}.
* The state is persisted automatically in regular intervals, but calling this method manually
@@ -174,7 +174,7 @@ export class RequestList {
*
* @return {Promise<void>}
*/
- persistState(): Promise<void>;
+ async persistState(): Promise<void>;
/**
* Unlike persistState(), this is used only internally, since the sources
* are automatically persisted at RequestList initialization (if the persistSourcesKey is set),
@@ -183,7 +183,7 @@ export class RequestList {
* @return {Promise<void>}
* @ignore
*/
- _persistSources(): Promise<void>;
+ async _persistSources(): Promise<void>;
/**
* Restores RequestList state from a state object.
*
@@ -198,7 +198,7 @@ export class RequestList {
* @return {Promise<Array>}
* @ignore
*/
- _loadStateAndSources(): Promise<any[]>;
+ async _loadStateAndSources(): Promise<any[]>;
/**
* Returns an object representing the internal state of the `RequestList` instance.
* Note that the object's fields can change in future releases.
@@ -213,13 +213,13 @@ export class RequestList {
*
* @returns {Promise<Boolean>}
*/
- isEmpty(): Promise<boolean>;
+ async isEmpty(): Promise<boolean>;
/**
* Returns `true` if all requests were already handled and there are no more left.
*
* @returns {Promise<Boolean>}
*/
- isFinished(): Promise<boolean>;
+ async isFinished(): Promise<boolean>;
/**
* Gets the next {@link Request} to process. First, the function gets a request previously reclaimed
* using the {@link RequestList#reclaimRequest} function, if there is any.
@@ -230,7 +230,7 @@ export class RequestList {
*
* @returns {Promise<Request>}
*/
- fetchNextRequest(): Promise<Request>;
+ async fetchNextRequest(): Promise<Request>;
/**
* Marks request as handled after successful processing.
*
@@ -238,7 +238,7 @@ export class RequestList {
*
* @returns {Promise<void>}
*/
- markRequestHandled(request: Request): Promise<void>;
+ async markRequestHandled(request: Request): Promise<void>;
/**
* Reclaims request to the list if its processing failed.
* The request will become available in the next `this.fetchNextRequest()`.
@@ -247,20 +247,20 @@ export class RequestList {
*
* @returns {Promise<void>}
*/
- reclaimRequest(request: Request): Promise<void>;
+ async reclaimRequest(request: Request): Promise<void>;
/**
* Adds all fetched requests from a URL from a remote resource.
*
* @ignore
*/
- _addFetchedRequests(source: any, fetchedRequests: any): Promise<void>;
+ async _addFetchedRequests(source: any, fetchedRequests: any): Promise<void>;
/**
* Fetches URLs from requestsFromUrl and returns them in format of list of requests
* @param source
* @return {Promise<RequestOptions[]>}
* @ignore
*/
- _fetchRequestsFromUrl(source: any): Promise<any[]>;
+ async _fetchRequestsFromUrl(source: any): Promise<any[]>;
/**
* Adds given request.
* If the `opts` parameter is a plain object and not an instance of a `Request`, then the function
diff --git a/types/request_queue.d.ts b/types/request_queue.d.ts
index 109383bb8..ea3071d7e 100644
--- a/types/request_queue.d.ts
+++ b/types/request_queue.d.ts
@@ -112,7 +112,7 @@ export class RequestQueue {
* @param {Boolean} [options.forefront=false] If `true`, the request will be added to the foremost position in the queue.
* @return {Promise<QueueOperationInfo>}
*/
- addRequest(request: RequestOptions | Request, options?: {
+ async addRequest(request: RequestOptions | Request, options?: {
forefront?: boolean;
}): Promise<QueueOperationInfo>;
/**
@@ -121,7 +121,7 @@ export class RequestQueue {
* @param {String} requestId ID of the request.
* @return {Promise<Request>} Returns the request object, or `null` if it was not found.
*/
- getRequest(requestId: string): Promise<Request>;
+ async getRequest(requestId: string): Promise<Request>;
/**
* Returns a next request in the queue to be processed, or `null` if there are no more pending requests.
*
@@ -139,7 +139,7 @@ export class RequestQueue {
* @returns {Promise<Request>}
* Returns the request object or `null` if there are no more pending requests.
*/
- fetchNextRequest(): Promise<Request>;
+ async fetchNextRequest(): Promise<Request>;
/**
* Marks a request that was previously returned by the
* {@link RequestQueue#fetchNextRequest}
@@ -149,7 +149,7 @@ export class RequestQueue {
* @param {Request} request
* @return {Promise<QueueOperationInfo>}
*/
- markRequestHandled(request: Request): Promise<QueueOperationInfo>;
+ async markRequestHandled(request: Request): Promise<QueueOperationInfo>;
/**
* Reclaims a failed request back to the queue, so that it can be returned for processed later again
* by another call to {@link RequestQueue#fetchNextRequest}.
@@ -164,7 +164,7 @@ export class RequestQueue {
* By default, it's put to the end of the queue.
* @return {Promise<QueueOperationInfo>}
*/
- reclaimRequest(request: Request, options?: {
+ async reclaimRequest(request: Request, options?: {
forefront?: boolean;
}): Promise<QueueOperationInfo>;
/**
@@ -175,7 +175,7 @@ export class RequestQueue {
*
* @returns {Promise<Boolean>}
*/
- isEmpty(): Promise<boolean>;
+ async isEmpty(): Promise<boolean>;
/**
* Resolves to `true` if all requests were already handled and there are no more left.
* Due to the nature of distributed storage used by the queue,
@@ -184,7 +184,7 @@ export class RequestQueue {
*
* @returns {Promise<Boolean>}
*/
- isFinished(): Promise<boolean>;
+ async isFinished(): Promise<boolean>;
/**
* Caches information about request to beware of unneeded addRequest() calls.
*
@@ -202,21 +202,21 @@ export class RequestQueue {
* @return {Boolean} Indicates if queue head is consistent (true) or inconsistent (false).
* @ignore
*/
- _ensureHeadIsNonEmpty(ensureConsistency?: boolean, limit?: number, iteration?: number): boolean;
+ async _ensureHeadIsNonEmpty(ensureConsistency?: boolean, limit?: number, iteration?: number): boolean;
/**
* Adds a request straight to the queueHeadDict, to improve performance.
* @private
*/
- _maybeAddRequestToQueueHead(requestId: any, forefront: any): void;
+ private _maybeAddRequestToQueueHead;
/**
* Removes the queue either from the Apify Cloud storage or from the local directory,
* depending on the mode of operation.
*
* @return {Promise}
*/
- drop(): Promise<any>;
+ async drop(): Promise<any>;
/** @ignore */
- delete(): Promise<void>;
+ async delete(): Promise<void>;
/**
* Returns the number of handled requests.
*
@@ -228,7 +228,7 @@ export class RequestQueue {
*
* @return {Promise<number>}
*/
- handledCount(): Promise<number>;
+ async handledCount(): Promise<number>;
/**
* Returns an object containing general information about the request queue.
*
@@ -255,7 +255,7 @@ export class RequestQueue {
*
* @returns {Promise<Object>}
*/
- getInfo(): Promise<any>;
+ async getInfo(): Promise<any>;
}
/**
* Local directory-based implementation of the `RequestQueue` class.
@@ -282,37 +282,37 @@ export class RequestQueueLocal {
modifiedAt: Date;
accessedAt: Date;
initializationPromise: Promise<void>;
- _initialize(): Promise<void>;
- _saveRequestIdToQueueOrderNo(filepath: any): Promise<void>;
+ async _initialize(): Promise<void>;
+ async _saveRequestIdToQueueOrderNo(filepath: any): Promise<void>;
_getFilePath(queueOrderNo: any, isHandled?: boolean): string;
_getQueueOrderNo(forefront?: boolean): number;
- _getRequestByQueueOrderNo(queueOrderNo: any): Promise<Request>;
- addRequest(request: any, opts?: {}): Promise<{
+ async _getRequestByQueueOrderNo(queueOrderNo: any): Promise<Request>;
+ async addRequest(request: any, opts?: {}): Promise<{
requestId: any;
wasAlreadyHandled: any;
wasAlreadyPresent: boolean;
request: any;
}>;
- getRequest(requestId: any): Promise<any>;
- fetchNextRequest(): Promise<Request>;
- markRequestHandled(request: any): Promise<{
+ async getRequest(requestId: any): Promise<any>;
+ async fetchNextRequest(): Promise<Request>;
+ async markRequestHandled(request: any): Promise<{
requestId: any;
wasAlreadyHandled: boolean;
wasAlreadyPresent: boolean;
request: any;
}>;
- reclaimRequest(request: any, opts?: {}): Promise<{
+ async reclaimRequest(request: any, opts?: {}): Promise<{
requestId: any;
wasAlreadyHandled: boolean;
wasAlreadyPresent: boolean;
request: any;
}>;
- isEmpty(): Promise<boolean>;
- isFinished(): Promise<boolean>;
- drop(): Promise<void>;
- delete(): Promise<void>;
- handledCount(): Promise<number>;
- getInfo(): Promise<{
+ async isEmpty(): Promise<boolean>;
+ async isFinished(): Promise<boolean>;
+ async drop(): Promise<void>;
+ async delete(): Promise<void>;
+ async handledCount(): Promise<number>;
+ async getInfo(): Promise<{
id: any;
name: any;
userId: string;
diff --git a/types/session_pool/session.d.ts b/types/session_pool/session.d.ts
index 6ecb6f136..023b67ba0 100644
--- a/types/session_pool/session.d.ts
+++ b/types/session_pool/session.d.ts
@@ -46,7 +46,7 @@ export class Session {
* @type CookieJar
* @private
* */
- cookieJar: CookieJar;
+ private cookieJar;
id: string;
maxAgeSecs: number;
userData: any;
@@ -164,21 +164,21 @@ export class Session {
* @return {Cookie}
* @private
*/
- _puppeteerCookieToTough(puppeteerCookie: PuppeteerCookie): Cookie;
+ private _puppeteerCookieToTough;
/**
* Transforms tough-cookie to puppeteer cookie .
* @param {Cookie} toughCookie - Cookie from CookieJar
* @return {PuppeteerCookie} - Cookie from Puppeteer
* @private
*/
- _toughCookieToPuppeteer(toughCookie: Cookie): PuppeteerCookie;
+ private _toughCookieToPuppeteer;
/**
* Sets cookies.
* @param {Cookie[]} cookies
* @param {string} url
* @private
*/
- _setCookies(cookies: Cookie[], url: string): void;
+ private _setCookies;
}
/**
* Persistable {@link Session} state.
@@ -246,7 +246,5 @@ export type SessionOptions = {
*/
sessionPool: SessionPool;
};
-import { CookieJar } from "tough-cookie";
import { SessionPool } from "./session_pool";
import { Cookie as PuppeteerCookie } from "puppeteer";
-import { Cookie } from "tough-cookie";
diff --git a/types/session_pool/session_pool.d.ts b/types/session_pool/session_pool.d.ts
index 74b837d2d..487b3b09f 100644
--- a/types/session_pool/session_pool.d.ts
+++ b/types/session_pool/session_pool.d.ts
@@ -68,9 +68,9 @@ export class SessionPool extends EventEmitter {
*/
constructor(options?: SessionPoolOptions);
maxPoolSize: number;
- createSessionFunction: any;
+ createSessionFunction: CreateSession;
sessionOptions: {};
- persistStateKeyValueStoreId: any;
+ persistStateKeyValueStoreId: string;
persistStateKey: string;
keyValueStore: import("../key_value_store").KeyValueStore;
sessions: any[];
@@ -90,7 +90,7 @@ export class SessionPool extends EventEmitter {
*
* @return {Promise<void>}
*/
- initialize(): Promise<void>;
+ async initialize(): Promise<void>;
_listener: any;
/**
* Gets session.
@@ -100,7 +100,7 @@ export class SessionPool extends EventEmitter {
*
* @return {Promise<Session>}
*/
- getSession(): Promise<Session>;
+ async getSession(): Promise<Session>;
/**
* Returns an object representing the internal state of the `SessionPool` instance.
* Note that the object's fields can change in future releases.
@@ -114,7 +114,7 @@ export class SessionPool extends EventEmitter {
*
* @return {Promise}
*/
- persistState(): Promise<any>;
+ async persistState(): Promise<any>;
/**
* Removes listener from `persistState` event.
* This function should be called after you are done with using the `SessionPool` instance.
@@ -125,51 +125,51 @@ export class SessionPool extends EventEmitter {
* @param session {Session} - Session to be removed
* @private
*/
- _removeSession(session: Session): void;
+ private _removeSession;
/**
* Adds `Session` instance to `SessionPool`.
* @param newSession {Session} - `Session` instance to be added.
* @private
*/
- _addSession(newSession: Session): void;
+ private _addSession;
/**
* Gets random index.
* @return {number}
* @private
*/
- _getRandomIndex(): number;
+ private _getRandomIndex;
/**
* Creates new session without any extra behavior.
* @param sessionPool
* @return {Session} - New session.
* @private
*/
- _defaultCreateSessionFunction(sessionPool: any): Session;
+ private _defaultCreateSessionFunction;
/**
* Creates new session and adds it to the pool.
* @return {Promise<Session>} - Newly created `Session` instance.
* @private
*/
- _createSession(): Promise<Session>;
+ private async _createSession;
/**
* Decides whether there is enough space for creating new session.
* @return {boolean}
* @private
*/
- _hasSpaceForSession(): boolean;
+ private _hasSpaceForSession;
/**
* Picks random session from the `SessionPool`.
* @return {Session} - Picked `Session`
* @private
*/
- _pickSession(): Session;
+ private _pickSession;
/**
* Potentially loads `SessionPool`.
* If the state was persisted it loads the `SessionPool` from the persisted state.
* @return {Promise<void>}
* @private
*/
- _maybeLoadSessionPool(): Promise<void>;
+ private async _maybeLoadSessionPool;
addListener(event: string | symbol, listener: (...args: any[]) => void): SessionPool;
on(event: string | symbol, listener: (...args: any[]) => void): SessionPool;
once(event: string | symbol, listener: (...args: any[]) => void): SessionPool;