Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Remove bluebird dependency

Ondra Urban14de9699da1e
diff --git a/package.json b/package.json
index aea649b9a..e079855f2 100644
--- a/package.json
+++ b/package.json
@@ -52,7 +52,6 @@
"@apify/ps-tree": "^1.1.3",
"apify-client": "^0.5.14",
"apify-shared": "^0.1.41",
- "bluebird": "^3.5.3",
"cheerio": "^1.0.0-rc.2",
"content-type": "^1.0.4",
"express": "^4.16.4",
diff --git a/src/actor.js b/src/actor.js
index 822c503bc..066edc1f1 100644
--- a/src/actor.js
+++ b/src/actor.js
@@ -1,6 +1,5 @@
import path from 'path';
import _ from 'underscore';
-import Promise from 'bluebird';
import log from 'apify-shared/log';
import { checkParamOrThrow } from 'apify-client/build/utils';
import { APIFY_PROXY_VALUE_REGEX } from 'apify-shared/regexs';
@@ -55,7 +54,7 @@ const waitForRunToFinish = async ({ actId, runId, token, waitSecs, taskId }) =>
// It might take some time for database replicas to get up-to-date,
// so getRun() might return null. Wait a little bit and try it again.
- if (!updatedRun) await Promise.delay(250);
+ if (!updatedRun) await sleep(250);
}
if (!updatedRun) {
diff --git a/src/dataset.js b/src/dataset.js
index 1fe20fe76..898b380db 100644
--- a/src/dataset.js
+++ b/src/dataset.js
@@ -1,8 +1,7 @@
-import fs from 'fs';
-import fsExtra from 'fs-extra';
import path from 'path';
+import { promisify } from 'util';
+import fs from 'fs-extra';
import _ from 'underscore';
-import Promise from 'bluebird';
import { leftpad } from 'apify-shared/utilities';
import LruCache from 'apify-shared/lru_cache';
import { checkParamOrThrow } from 'apify-client/build/utils';
@@ -15,11 +14,11 @@ export const LOCAL_GET_ITEMS_DEFAULT_LIMIT = 250000;
const MAX_OPENED_STORES = 1000;
const SAFETY_BUFFER_PERCENT = 0.01 / 100; // 0.01%
-const writeFilePromised = Promise.promisify(fs.writeFile);
-const readFilePromised = Promise.promisify(fs.readFile);
-const readdirPromised = Promise.promisify(fs.readdir);
-const statPromised = Promise.promisify(fs.stat);
-const emptyDirPromised = Promise.promisify(fsExtra.emptyDir);
+const writeFilePromised = promisify(fs.writeFile);
+const readFilePromised = promisify(fs.readFile);
+const readdirPromised = promisify(fs.readdir);
+const statPromised = promisify(fs.stat);
+const emptyDirPromised = promisify(fs.emptyDir);
const getLocaleFilename = index => `${leftpad(index, LOCAL_FILENAME_DIGITS, 0)}.json`;
@@ -178,32 +177,25 @@ export class Dataset {
* The objects must be serializable to JSON and the JSON representation of each object must be smaller than 9MB.
* @return {Promise}
*/
- pushData(data) {
+ async pushData(data) {
checkParamOrThrow(data, 'data', 'Array | Object');
- const dispatch = payload => datasets.putItems({ datasetId: this.datasetId, data: payload });
+ const dispatch = async payload => datasets.putItems({ datasetId: this.datasetId, data: payload });
const limit = MAX_PAYLOAD_SIZE_BYTES - Math.ceil(MAX_PAYLOAD_SIZE_BYTES * SAFETY_BUFFER_PERCENT);
// Handle singular Objects
if (!Array.isArray(data)) {
- try {
- const payload = checkAndSerialize(data, limit);
- return dispatch(payload);
- } catch (err) {
- return Promise.reject(err);
- }
+ const payload = checkAndSerialize(data, limit);
+ return dispatch(payload);
}
// Handle Arrays
- let payloads;
- try {
- payloads = data.map((item, index) => checkAndSerialize(item, limit, index));
- } catch (err) {
- return Promise.reject(err);
- }
+ const payloads = data.map((item, index) => checkAndSerialize(item, limit, index));
const chunks = chunkBySize(payloads, limit);
// Invoke client in series to preserve order of data
- return Promise.mapSeries(chunks, chunk => dispatch(chunk));
+ for (const chunk of chunks) {
+ await dispatch(chunk);
+ }
}
/**
@@ -307,27 +299,23 @@ export class Dataset {
* @param {Number} [index=0] Specifies the initial index number passed to the `iteratee` function.
* @return {Promise}
*/
- forEach(iteratee, options = {}, index = 0) {
+ async forEach(iteratee, options = {}, index = 0) {
if (!options.offset) options.offset = 0;
if (options.format && options.format !== 'json') throw new Error('Dataset.forEach/map/reduce() support only a "json" format.');
- return this
- .getData(options)
- .then(({ items, total, limit, offset }) => {
- return Promise
- .mapSeries(items, item => iteratee(item, index++))
- .then(() => {
- const newOffset = offset + limit;
+ const { items, total, limit, offset } = await this.getData(options);
- if (newOffset >= total) return undefined;
+ for (const item of items) {
+ await iteratee(item, index++);
+ }
- const newOpts = Object.assign({}, options, {
- offset: newOffset,
- });
+ const newOffset = offset + limit;
+ if (newOffset >= total) return;
- return this.forEach(iteratee, newOpts, index);
- });
- });
+ const newOpts = Object.assign({}, options, {
+ offset: newOffset,
+ });
+ return this.forEach(iteratee, newOpts, index);
}
/**
@@ -483,7 +471,7 @@ export class DatasetLocal {
});
}
- getData(opts = {}) {
+ async getData(opts = {}) {
checkParamOrThrow(opts, 'opts', 'Object');
checkParamOrThrow(opts.limit, 'opts.limit', 'Maybe Number');
checkParamOrThrow(opts.offset, 'opts.offset', 'Maybe Number');
@@ -491,22 +479,22 @@ export class DatasetLocal {
if (!opts.limit) opts.limit = LOCAL_GET_ITEMS_DEFAULT_LIMIT;
if (!opts.offset) opts.offset = 0;
- return this.initializationPromise
- .then(() => {
- const indexes = this._getItemIndexes(opts.offset, opts.limit);
+ await this.initializationPromise;
+ const indexes = this._getItemIndexes(opts.offset, opts.limit);
+ const items = [];
+ for (const idx of indexes) {
+ const item = await this._readAndParseFile(idx);
+ items.push(item);
+ }
- return Promise.mapSeries(indexes, index => this._readAndParseFile(index));
- })
- .then((items) => {
- this._updateMetadata();
- return {
- items,
- total: this.counter,
- offset: opts.offset,
- count: items.length,
- limit: opts.limit,
- };
- });
+ this._updateMetadata();
+ return {
+ items,
+ total: this.counter,
+ offset: opts.offset,
+ count: items.length,
+ limit: opts.limit,
+ };
}
async getInfo() {
@@ -530,46 +518,35 @@ export class DatasetLocal {
return result;
}
- forEach(iteratee) {
- return this.initializationPromise
- .then(() => {
- const indexes = this._getItemIndexes();
-
- return Promise.each(indexes, (index) => {
- return this
- ._readAndParseFile(index)
- .then(item => iteratee(item, index - 1));
- });
- })
- .then(() => undefined);
+ async forEach(iteratee) {
+ await this.initializationPromise;
+ const indexes = this._getItemIndexes();
+ for (const idx of indexes) {
+ const item = await this._readAndParseFile(idx);
+ await iteratee(item, idx - 1);
+ }
}
- map(iteratee) {
- return this.initializationPromise
- .then(() => {
- const indexes = this._getItemIndexes();
-
- return Promise
- .map(indexes, (index) => {
- return this
- ._readAndParseFile(index)
- .then(item => iteratee(item, index - 1));
- });
- });
+ async map(iteratee) {
+ await this.initializationPromise;
+ const indexes = this._getItemIndexes();
+ const results = [];
+ for (const idx of indexes) {
+ const item = await this._readAndParseFile(idx);
+ const result = await iteratee(item, idx - 1);
+ results.push(result);
+ }
+ return results;
}
- reduce(iteratee, memo) {
- return this.initializationPromise
- .then(() => {
- const indexes = this._getItemIndexes();
-
- return Promise
- .reduce(indexes, (currentMemo, index) => {
- return this
- ._readAndParseFile(index)
- .then(item => iteratee(currentMemo, item, index - 1));
- }, memo);
- });
+ async reduce(iteratee, memo) {
+ await this.initializationPromise;
+ const indexes = this._getItemIndexes();
+ for (const idx of indexes) {
+ const item = await this._readAndParseFile(idx);
+ memo = await iteratee(memo, item, idx - 1);
+ }
+ return memo;
}
delete() {
@@ -585,7 +562,7 @@ export class DatasetLocal {
* Returns an array of item indexes for given offset and limit.
*/
_getItemIndexes(offset = 0, limit = this.counter) {
- if (limit === null) throw new Error('DatasetLocal must be initialize before calling this._getItemIndexes()!');
+ if (limit === null) throw new Error('DatasetLocal must be initialized before calling this._getItemIndexes()!');
const start = offset + 1;
const end = Math.min(offset + limit, this.counter) + 1;
if (start > end) return [];
diff --git a/src/key_value_store.js b/src/key_value_store.js
index 998e25685..78a1e17ec 100644
--- a/src/key_value_store.js
+++ b/src/key_value_store.js
@@ -1,10 +1,9 @@
-import fs from 'fs';
-import fsExtra from 'fs-extra';
+import fs from 'fs-extra';
import path from 'path';
-import Promise from 'bluebird';
+import { promisify } from 'util';
import contentTypeParser from 'content-type';
-import LruCache from 'apify-shared/lru_cache';
import mime from 'mime';
+import LruCache from 'apify-shared/lru_cache';
import { KEY_VALUE_STORE_KEY_REGEX } from 'apify-shared/regexs';
import { ENV_VARS, LOCAL_STORAGE_SUBDIRS, KEY_VALUE_STORE_KEYS } from 'apify-shared/consts';
import { checkParamOrThrow, parseBody } from 'apify-client/build/utils';
@@ -18,12 +17,12 @@ const MAX_OPENED_STORES = 1000;
const DEFAULT_LOCAL_FILE_EXTENSION = 'bin';
const COMMON_LOCAL_FILE_EXTENSIONS = ['bin', 'txt', 'json', 'html', 'xml', 'jpeg', 'png', 'pdf', 'mp3', 'js', 'css', 'csv'];
-const readFilePromised = Promise.promisify(fs.readFile);
-const readdirPromised = Promise.promisify(fs.readdir);
-const writeFilePromised = Promise.promisify(fs.writeFile);
-const unlinkPromised = Promise.promisify(fs.unlink);
-const statPromised = Promise.promisify(fs.stat);
-const emptyDirPromised = Promise.promisify(fsExtra.emptyDir);
+const readFilePromised = promisify(fs.readFile);
+const readdirPromised = promisify(fs.readdir);
+const writeFilePromised = promisify(fs.writeFile);
+const unlinkPromised = promisify(fs.unlink);
+const statPromised = promisify(fs.stat);
+const emptyDirPromised = promisify(fs.emptyDir);
const { keyValueStores } = apifyClient;
const storesCache = new LruCache({ maxLength: MAX_OPENED_STORES }); // Open key-value stores are stored here.
@@ -483,24 +482,19 @@ export class KeyValueStoreLocal {
* }
* @ignore
*/
- _handleFile(key, handler) {
- return Promise.map(COMMON_LOCAL_FILE_EXTENSIONS, (extension) => {
+ async _handleFile(key, handler) {
+ for (const extension of COMMON_LOCAL_FILE_EXTENSIONS) {
const fileName = `${key}.${extension}`;
const filePath = this._getPath(fileName);
- return handler(filePath)
- .then(returnValue => ({ returnValue, fileName }))
- .catch((err) => {
- if (err.code === 'ENOENT') return null;
- throw err;
- });
- })
- .then((results) => {
- // Using filter here to distinguish between no result and undefined result. [] vs [undefined]
- const result = results.filter(r => r && r.returnValue !== null);
- return result.length
- ? result[0]
- : this._fullDirectoryLookup(key, handler);
- });
+ try {
+ const returnValue = await handler(filePath);
+ return { returnValue, fileName };
+ } catch (err) {
+ if (err.code !== 'ENOENT') throw err;
+ }
+ }
+
+ return this._fullDirectoryLookup(key, handler);
}
/**
diff --git a/src/request_queue.js b/src/request_queue.js
index 1dac54864..05e51c084 100644
--- a/src/request_queue.js
+++ b/src/request_queue.js
@@ -1,16 +1,15 @@
-import fs from 'fs';
-import fsExtra from 'fs-extra';
import path from 'path';
+import crypto from 'crypto';
+import { promisify } from 'util';
+import fs from 'fs-extra';
import { checkParamOrThrow } from 'apify-client/build/utils';
import LruCache from 'apify-shared/lru_cache';
import ListDictionary from 'apify-shared/list_dictionary';
import { ENV_VARS, LOCAL_STORAGE_SUBDIRS, REQUEST_QUEUE_HEAD_MAX_LIMIT } from 'apify-shared/consts';
-import { delayPromise, checkParamPrototypeOrThrow } from 'apify-shared/utilities';
+import { checkParamPrototypeOrThrow } from 'apify-shared/utilities';
import log from 'apify-shared/log';
-import Promise from 'bluebird';
-import crypto from 'crypto';
import Request from './request';
-import { ensureDirExists, apifyClient, openRemoteStorage, openLocalStorage, ensureTokenOrLocalStorageEnvExists } from './utils';
+import { sleep, ensureDirExists, apifyClient, openRemoteStorage, openLocalStorage, ensureTokenOrLocalStorageEnvExists } from './utils';
export const LOCAL_STORAGE_SUBDIR = LOCAL_STORAGE_SUBDIRS.requestQueues;
const MAX_OPENED_QUEUES = 1000;
@@ -27,12 +26,12 @@ export const API_PROCESSED_REQUESTS_DELAY_MILLIS = 10 * 1000;
// How many times we try to get queue head with queueModifiedAt older than API_PROCESSED_REQUESTS_DELAY_MILLIS.
export const MAX_QUERIES_FOR_CONSISTENCY = 6;
-const writeFilePromised = Promise.promisify(fs.writeFile);
-const readdirPromised = Promise.promisify(fs.readdir);
-const readFilePromised = Promise.promisify(fs.readFile);
-const renamePromised = Promise.promisify(fs.rename);
-const statPromised = Promise.promisify(fs.stat);
-const emptyDirPromised = Promise.promisify(fsExtra.emptyDir);
+const writeFilePromised = promisify(fs.writeFile);
+const readdirPromised = promisify(fs.readdir);
+const readFilePromised = promisify(fs.readFile);
+const renamePromised = promisify(fs.rename);
+const statPromised = promisify(fs.stat);
+const emptyDirPromised = promisify(fs.emptyDir);
const { requestQueues } = apifyClient;
const queuesCache = new LruCache({ maxLength: MAX_OPENED_QUEUES }); // Open queues are stored here.
@@ -527,7 +526,7 @@ export class RequestQueue {
? API_PROCESSED_REQUESTS_DELAY_MILLIS
: 0;
- return delayPromise(delayMillis)
+ return sleep(delayMillis)
.then(() => this._ensureHeadIsNonEmpty(checkModifiedAt, nextLimit, iteration + 1));
});
}
@@ -655,7 +654,12 @@ export class RequestQueueLocal {
this.modifiedAt = stats.mtime;
this.accessedAt = stats.atime;
- return Promise.mapSeries(filePaths, filepath => this._readFile(filepath));
+ const files = [];
+ for (const filePath of filePaths) {
+ const file = await this._readFile(filePath);
+ files.push(file);
+ }
+ return files;
}
async _readFile(filepath) {
diff --git a/src/utils.js b/src/utils.js
index 15054861f..96bfe20f5 100644
--- a/src/utils.js
+++ b/src/utils.js
@@ -10,7 +10,7 @@ import requestPromise from 'request-promise-native';
import XRegExp from 'xregexp';
import cheerio from 'cheerio';
import log from 'apify-shared/log';
-import { delayPromise, getRandomInt } from 'apify-shared/utilities';
+import { getRandomInt } from 'apify-shared/utilities';
import { ENV_VARS, LOCAL_ENV_VARS } from 'apify-shared/consts';
import { checkParamOrThrow } from 'apify-client/build/utils';
import { version as apifyClientVersion } from 'apify-client/package.json';
@@ -352,18 +352,6 @@ export const addTimeoutToPromise = (promise, timeoutMillis, errorMessage) => {
});
};
-/**
- * Creates a promise that after given time gets rejected with given error.
- *
- * @return {Promise<Error>}
- * @ignore
- */
-export const createTimeoutPromise = (timeoutMillis, errorMessage) => {
- return delayPromise(timeoutMillis).then(() => {
- throw new Error(errorMessage);
- });
-};
-
/**
* Returns `true` when code is running on Apify platform and `false` otherwise (for example locally).
*
@@ -395,7 +383,7 @@ export const isAtHome = () => !!process.env[ENV_VARS.IS_AT_HOME];
* @return {Promise}
*/
export const sleep = (millis) => {
- return delayPromise(millis);
+ return new Promise(res => setTimeout(res, millis));
};
/**