Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Updating docs
Marek Trunkat05feee1d4000
diff --git a/src/dataset.js b/src/dataset.jsindex ca2527fbd..fd504ea59 100644--- a/src/dataset.js+++ b/src/dataset.js@@ -104,14 +104,14 @@ export const chunkBySize = (items, limitBytes) => {* @property {Number} limit - Requested limit*/-/*** The `Dataset` class represents an append-only data storage that is useful for saving sequential or tabular data,* such as a list of e-commerce products.* To create an instance of the `Dataset` class, call the [Apify.openDataset()](#module-Apify-openDataset) function.*- * The actual data is either stored in the Apify cloud (see [Dataset storage documentation](https://www.apify.com/docs/storage#dataset),- * or on the local disk in the directory specified by the `APIFY_LOCAL_EMULATION_DIR` environment variable (if set).+ * The actual data is either stored in the Apify cloud (see [Dataset storage documentation](https://www.apify.com/docs/storage#dataset)+ * when actor is running on Apify platform or `APIFY_PLATFORM_STORAGE=1` environment variable is set+ * or on the local disk in the directory `./apify_storage` (overridable by `APIFY_LOCAL_STORAGE_DIR` environment variable).** Example usage:*@@ -381,19 +381,19 @@ export class Dataset {* @ignore*/export class DatasetLocal {- constructor(datasetId, localEmulationDir) {+ constructor(datasetId, localStorageDir) {checkParamOrThrow(datasetId, 'datasetId', 'String');- checkParamOrThrow(localEmulationDir, 'localEmulationDir', 'String');+ checkParamOrThrow(localStorageDir, 'localStorageDir', 'String');- this.localEmulationPath = path.resolve(path.join(localEmulationDir, LOCAL_STORAGE_SUBDIR, datasetId));+ this.localStoragePath = path.resolve(path.join(localStorageDir, LOCAL_STORAGE_SUBDIR, datasetId));this.counter = null;this.datasetId = datasetId;this.initializationPromise = this._initialize();}_initialize() {- return ensureDirExists(this.localEmulationPath)- .then(() => readdirPromised(this.localEmulationPath))+ return ensureDirExists(this.localStoragePath)+ .then(() => readdirPromised(this.localStoragePath)).then((files) => {if (files.length) {const lastFileNum = files.pop().split('.')[0];@@ -401,7 +401,7 @@ export class DatasetLocal {} else {this.counter = 0;}- return statPromised(this.localEmulationPath);+ return statPromised(this.localStoragePath);}).then((stats) => {this.createdAt = stats.birthtime;@@ -422,7 +422,7 @@ export class DatasetLocal {// Format JSON to simplify debugging, the overheads is negligibleconst itemStr = JSON.stringify(item, null, 2);- const filePath = path.join(this.localEmulationPath, getLocaleFilename(this.counter));+ const filePath = path.join(this.localStoragePath, getLocaleFilename(this.counter));return writeFilePromised(filePath, itemStr);});@@ -518,7 +518,7 @@ export class DatasetLocal {delete() {return this.initializationPromise- .then(() => emptyDirPromised(this.localEmulationPath))+ .then(() => emptyDirPromised(this.localStoragePath)).then(() => {this._updateMetadata(true);datasetsCache.remove(this.datasetId);@@ -541,7 +541,7 @@ export class DatasetLocal {* Reads and parses file for given index.*/_readAndParseFile(index) {- const filePath = path.join(this.localEmulationPath, getLocaleFilename(index));+ const filePath = path.join(this.localStoragePath, getLocaleFilename(index));return readFilePromised(filePath).then((json) => {@@ -633,8 +633,9 @@ export const openDataset = (datasetIdOrName) => {* await dataset.pushData({ myValue: 123 });* ```*- * The actual data is either stored in the Apify cloud (see [Dataset storage documentation](https://www.apify.com/docs/storage#dataset),- * or on the local disk in the directory specified by the `APIFY_LOCAL_EMULATION_DIR` environment variable (if set).+ * The actual data is either stored in the Apify cloud (see [Dataset storage documentation](https://www.apify.com/docs/storage#dataset)+ * when actor is running on Apify platform or `APIFY_PLATFORM_STORAGE=1` environment variable is set+ * or on the local disk in the directory `./apify_storage` (overridable by `APIFY_LOCAL_STORAGE_DIR` environment variable) otherwise.** For more information, see* [Apify.openDataset()](#module-Apify-openDataset) and [Dataset.pushData()](#Dataset-pushData).diff --git a/src/index.js b/src/index.jsindex 8a16f7a93..9420f0607 100644--- a/src/index.js+++ b/src/index.js@@ -27,7 +27,7 @@ if (!isProduction() || process.env[ENV_VARS.LOG_LEVEL] === 'DEBUG') log.isDebugM// Log as plain text not JSONlog.logJson = false;-// Just warn user that local emulation will be used.+// Just warn user that local storage will be used.if (process.env[ENV_VARS.PLATFORM_STORAGE]) {log.info(`Using local directory for storage as environment variable ${ENV_VARS.PLATFORM_STORAGE}=1 is not set.`);}diff --git a/src/key_value_store.js b/src/key_value_store.jsindex 7af70f6a2..d04ef7d9e 100644--- a/src/key_value_store.js+++ b/src/key_value_store.js@@ -94,10 +94,14 @@ export const maybeStringify = (value, options) => {/**- * The `KeyValueStore` class provides a simple interface to the [Apify Key-value stores](https://www.apify.com/docs/storage#kv-store).+ * The `KeyValueStore` class provides a simple interface to the [Apify Key-value stores](https://www.apify.com/docs/storage#key-value-store).* You should not instantiate this class directly, use the* [Apify.openKeyValueStore()](#module-Apify-openKeyValueStore) function.*+ * The actual data is either stored in the Apify cloud (see [Key-value stores documentation](https://www.apify.com/docs/storage#key-value-store)+ * when actor is running on Apify platform or `APIFY_PLATFORM_STORAGE=1` environment variable is set+ * or on the local disk in the directory `./apify_storage` (overridable by `APIFY_LOCAL_STORAGE_DIR` environment variable).+ ** Example usage:** ```javascript@@ -206,9 +210,9 @@ export class KeyValueStoreLocal {checkParamOrThrow(storeId, 'storeId', 'String');checkParamOrThrow(localStorageDir, 'localStorageDir', 'String');- this.localEmulationPath = path.resolve(path.join(localStorageDir, LOCAL_STORAGE_SUBDIR, storeId));+ this.localStoragePath = path.resolve(path.join(localStorageDir, LOCAL_STORAGE_SUBDIR, storeId));this.storeId = storeId;- this.initializationPromise = ensureDirExists(this.localEmulationPath);+ this.initializationPromise = ensureDirExists(this.localStoragePath);}getValue(key) {@@ -222,7 +226,7 @@ export class KeyValueStoreLocal {: null;}).catch((err) => {- throw new Error(`Error reading file '${key}' in directory '${this.localEmulationPath}' referred by ${ENV_VARS.LOCAL_STORAGE_DIR} environment variable: ${err.message}`); // eslint-disable-line max-len+ throw new Error(`Error reading file '${key}' in directory '${this.localStoragePath}' referred by ${ENV_VARS.LOCAL_STORAGE_DIR} environment variable: ${err.message}`); // eslint-disable-line max-len});}@@ -246,12 +250,12 @@ export class KeyValueStoreLocal {return deletePromise.then(() => writeFilePromised(filePath, value)).catch((err) => {- throw new Error(`Error writing file '${key}' in directory '${this.localEmulationPath}' referred by ${ENV_VARS.LOCAL_STORAGE_DIR} environment variable: ${err.message}`); // eslint-disable-line max-len+ throw new Error(`Error writing file '${key}' in directory '${this.localStoragePath}' referred by ${ENV_VARS.LOCAL_STORAGE_DIR} environment variable: ${err.message}`); // eslint-disable-line max-len});}delete() {- return emptyDirPromised(this.localEmulationPath)+ return emptyDirPromised(this.localStoragePath).then(() => {storesCache.remove(this.storeId);});@@ -302,7 +306,7 @@ export class KeyValueStoreLocal {* @ignore*/_fullDirectoryLookup(key, handler) {- return readdirPromised(this.localEmulationPath)+ return readdirPromised(this.localStoragePath).then((files) => {const regex = getFileNameRegexp(key);const fileName = files.find(file => regex.test(file));@@ -319,7 +323,7 @@ export class KeyValueStoreLocal {* @ignore*/_getPath(fileName) {- return path.resolve(this.localEmulationPath, fileName);+ return path.resolve(this.localStoragePath, fileName);}}@@ -348,6 +352,10 @@ const getOrCreateKeyValueStore = (storeIdOrName) => {* Key-value store is a simple storage for records, where each record has a unique key.* For more information, see [Key-value store documentation](https://www.apify.com/docs/storage#dataset).*+ * The actual data is either stored in the Apify cloud (see [Key-value stores documentation](https://www.apify.com/docs/storage#key-value-store)+ * when actor is running on Apify platform or `APIFY_PLATFORM_STORAGE=1` environment variable is set+ * or on the local disk in the directory `./apify_storage` (overridable by `APIFY_LOCAL_STORAGE_DIR` environment variable).+ ** Example usage:** ```javascript@@ -355,10 +363,6 @@ const getOrCreateKeyValueStore = (storeIdOrName) => {* await store.setValue('some-key', { foo: 'bar' });* ```*- * If the `APIFY_LOCAL_STORAGE_DIR` environment variable is set, the result of this function- * is an instance of the `KeyValueStoreLocal` class which stores the records in a local directory- * rather than Apify cloud. This is useful for local development and debugging of your actors.- ** @param {string} storeIdOrName ID or name of the key-value store to be opened. If no value is* provided then the function opens the default key-value store associated with the actor run.* @returns {Promise<KeyValueStore>} Returns a promise that resolves to a KeyValueStore object.@@ -398,13 +402,12 @@ export const openKeyValueStore = (storeIdOrName) => {* For all other content types, the body is a raw `Buffer`.* If the record cannot be found, the result is null.*- * If the `APIFY_LOCAL_STORAGE_DIR` environment variable is defined,- * the value is read from a that directory rather than the key-value store,- * specifically from a file that has the key as a name.- * file does not exists, the returned value is `null`. The file will get extension based on it's content type.+ * If the actor is not running on Apify platform and `APIFY_PLATFORM_STORAGE=1` environment variable is not set+ * the value is read from a local directory `./apify_storage` (overridable by `APIFY_LOCAL_STORAGE_DIR` environment variable)+ * rather than the Apify key-value store cloud storage, specifically from a file that has the key as a name.+ * If file does not exists, the returned value is `null`. The file will get extension based on it's content type.* This feature is useful for local development and debugging of your actors.*- ** @param {String} key Key of the record.* @returns {Promise} Returns a promise.*@@ -432,9 +435,11 @@ export const getValue = key => openKeyValueStore().then(store => store.getValue(* ```* In this case, the value must be a string or Buffer.*- * If the `APIFY_LOCAL_STORAGE_DIR` environment variable is defined,- * the value is written to that local directory rather than the key-value store on Apify cloud,- * to a file named as the key. This is useful for local development and debugging of your actors.+ * If the actor is not running on Apify platform and `APIFY_PLATFORM_STORAGE=1` environment variable is not set+ * the value is written to a local directory `./apify_storage` (overridable by `APIFY_LOCAL_STORAGE_DIR` environment variable)+ * rather than the Apify key-value store cloud storage, specifically to a file that has the key as a name.+ * The file will get extension based on it's content type.+ * This feature is useful for local development and debugging of your actors.** **IMPORTANT:** Do not forget to use the `await` keyword when calling `Apify.setValue()`,* otherwise the actor process might finish before the value is stored!diff --git a/src/request_queue.js b/src/request_queue.jsindex 642b55ede..9fa8ae884 100644--- a/src/request_queue.js+++ b/src/request_queue.js@@ -493,14 +493,14 @@ const filePathToQueueOrderNo = (filepath) => {* @ignore*/export class RequestQueueLocal {- constructor(queueId, localEmulationDir) {+ constructor(queueId, localStorageDir) {checkParamOrThrow(queueId, 'queueId', 'String');- checkParamOrThrow(localEmulationDir, 'localEmulationDir', 'String');+ checkParamOrThrow(localStorageDir, 'localStorageDir', 'String');this.queueId = queueId;- this.localEmulationPath = path.resolve(path.join(localEmulationDir, LOCAL_STORAGE_SUBDIR, queueId));- this.localHandledEmulationPath = path.join(this.localEmulationPath, 'handled');- this.localPendingEmulationPath = path.join(this.localEmulationPath, 'pending');+ this.localStoragePath = path.resolve(path.join(localStorageDir, LOCAL_STORAGE_SUBDIR, queueId));+ this.localHandledEmulationPath = path.join(this.localStoragePath, 'handled');+ this.localPendingEmulationPath = path.join(this.localStoragePath, 'pending');this.queueOrderNoCounter = 0; // Counter used in _getQueueOrderNo to ensure there won't be a collision.this.pendingCount = 0;@@ -512,7 +512,7 @@ export class RequestQueueLocal {}_initialize() {- return ensureDirExists(this.localEmulationPath)+ return ensureDirExists(this.localStoragePath).then(() => ensureDirExists(this.localHandledEmulationPath)).then(() => ensureDirExists(this.localPendingEmulationPath)).then(() => Promise.all([@@ -719,7 +719,7 @@ export class RequestQueueLocal {}delete() {- return emptyDirPromised(this.localEmulationPath)+ return emptyDirPromised(this.localStoragePath).then(() => {queuesCache.remove(this.queueId);});@@ -766,9 +766,9 @@ const getOrCreateQueue = (queueIdOrName) => {* queue.reclaimRequest(request2);* ```*- * If the `APIFY_LOCAL_STORAGE_DIR` environment variable is defined, the value this function- * returns is an instance of the `RequestQueueLocal` class which is a local emulation of the request queue.- * This is useful for local development and debugging of the actors.+ * The actual data is either stored in the Apify cloud (see [Request queue documentation](https://www.apify.com/docs/storage#queue)+ * when actor is running on Apify platform or `APIFY_PLATFORM_STORAGE=1` environment variable is set+ * or on the local disk in the directory `./apify_storage` (overridable by `APIFY_LOCAL_STORAGE_DIR` environment variable).** @param {string} queueIdOrName ID or name of the request queue to be opened.* @returns {Promise<RequestQueue>} Returns a promise that resolves to a `RequestQueue` object. If no value is provided