Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Added support for APIFY_MEMORY_MBYTES env var, updated docs
Jan Curnb2a33d0c6958
diff --git a/src/actor.js b/src/actor.jsindex 955734f42..45037a368 100644--- a/src/actor.js+++ b/src/actor.js@@ -27,7 +27,7 @@ const statPromised = Promise.promisify(fs.stat);/*** Tries to parse a string with date.* @param str Date string- * @return Returns either a Date object or undefined+ * @returns Returns either a Date object or undefined* @ignore*/const tryParseDate = (str) => {@@ -63,7 +63,7 @@ const getDefaultStoreIdOrThrow = () => {* This is useful for local development of the act.** @param callback Optional callback.- * @return Returns a promise if no callback was provided, otherwise the return value is not defined.+ * @returns Returns a promise if no callback was provided, otherwise the return value is not defined.*/export const getValue = (key, callback = null) => {if (!key || !_.isString(key)) throw new Error('The "key" parameter must be a non-empty string');@@ -155,7 +155,7 @@ export const getValue = (key, callback = null) => {* For any other value an error will be thrown.* @param options Optional settings, currently only \{ contentType: String \} is supported to set MIME content type of the value.* @param callback Optional callback.- * @return Returns a promise if no callback was provided, otherwise the return value is not defined.+ * @returns Returns a promise if no callback was provided, otherwise the return value is not defined.*/export const setValue = (key, value, options, callback = null) => {if (!key || !_.isString(key)) throw new Error('The "key" parameter must be a non-empty string');@@ -253,7 +253,8 @@ export const setValue = (key, value, options, callback = null) => {/*** @memberof module:Apify* @function- * @description Generates an object which contains parsed environment variables:+ * @description Returns a new object which contains information parsed from the `APIFY_XXX` environment variables.+ * It has the following fields:* ```javascript* {* actId: String,@@ -264,14 +265,16 @@ export const setValue = (key, value, options, callback = null) => {* timeoutAt: Date,* defaultKeyValueStoreId: String,* internalPort: Number,+ * memoryMbytes: Number,* }* ```- * All the information is generated from the APIFY_XXX environment variables.- * If some of the variables is not defined or is invalid, the corresponding value in the resulting object will be null;- * an error is not thrown in such a case in order to simplify local development and debugging of acts.- * @return Object+ * For the list of the `APIFY_XXX` environment variables, see+ * {@link http://localhost/docs/actor.php#run-env-vars|Actor documentation}.+ * If some of the variables is not defined or is invalid, the corresponding value in the resulting object will be null.+ * @returns {Object}*/export const getEnv = () => {+ // NOTE: don't throw if env vars are invalid to simplify local development and debugging of actsconst env = process.env || {};return {actId: env[ENV_VARS.ACT_ID] || null,@@ -282,6 +285,7 @@ export const getEnv = () => {timeoutAt: tryParseDate(env[ENV_VARS.TIMEOUT_AT]) || null,defaultKeyValueStoreId: env[ENV_VARS.DEFAULT_KEY_VALUE_STORE_ID] || null,internalPort: parseInt(env[ENV_VARS.INTERNAL_PORT], 10) || null,+ memoryMbytes: parseInt(env[ENV_VARS.MEMORY_MBYTES], 10) || null,};};@@ -289,9 +293,11 @@ export const getEnv = () => {* @memberof module:Apify* @function* @description To simplify development of acts, the runtime provides the `Apify.main(func)` function which does the following:- * 1) Invokes the user function `func`- * 2) If the function returned a promise, waits for it to resolve- * 3) Exits the process+ * <ol>+ * <li>Invokes the user function passed in the `func` parameter</li>+ * <li>If the function returned a promise, waits for it to resolve</li>+ * <li>Exits the process</li>+ * </ol>* If the user function throws an exception or some other error is encountered,* then `Apify.main()` prints the details to console so that they are stored to the log file.* `Apify.main()` accepts a single argument - the user function that performs the operation of the act.diff --git a/src/browser.js b/src/browser.jsindex b36edb1b8..c78ed1502 100644--- a/src/browser.js+++ b/src/browser.js@@ -87,7 +87,7 @@ export class Browser {/*** Initializes the browser.- * @return Promise+ * @returns Promise*/_initialize() {return newPromise()@@ -214,7 +214,7 @@ export const processBrowseArgs = (url, options, callback) => {* @param url Optional start URL to open. Defaults to about:blank* @param options Optional settings, their defaults are provided by the getDefaultBrowseOptions() function.* @param callback Optional callback.- * @return Returns a promise if no callback was provided, otherwise the return value is not defined.+ * @returns Returns a promise if no callback was provided, otherwise the return value is not defined.*/export const browse = (url, options, callback) => {const args = processBrowseArgs(url, options, callback);@@ -237,7 +237,7 @@ export const browse = (url, options, callback) => {// * This code is kept here for legacy reasons, it's not used.// * @param {boolean=} headless True (default) to launch Chrome in headless mode.// * Set to false to launch Chrome normally.-// * @return {Promise<ChromeLauncher>}+// * @returns {Promise<ChromeLauncher>}// */// export const launchChrome = (headless = !!process.env.APIFY_HEADLESS) => {// // code inspired by https://developers.google.com/web/updates/2017/04/headless-chromediff --git a/src/constants.js b/src/constants.jsindex cb331887e..7371359ee 100644--- a/src/constants.js+++ b/src/constants.js@@ -27,6 +27,7 @@ export const ENV_VARS = {API_BASE_URL: 'APIFY_API_BASE_URL',HEADLESS: 'APIFY_HEADLESS',INTERNAL_PORT: 'APIFY_INTERNAL_PORT',+ MEMORY_MBYTES: 'APIFY_MEMORY_MBYTES',};export const KEY_VALUE_STORE_KEYS = {diff --git a/src/proxy_chain.js b/src/proxy_chain.jsindex a5f53829c..f99d30239 100644--- a/src/proxy_chain.js+++ b/src/proxy_chain.js@@ -29,7 +29,7 @@ const fsWriteFilePromised = Promise.promisify(fs.writeFile);/** Run Squid process to get its version and wait for the finish- * @return Promise+ * @returns Promiseconst printSquidVersion = () => {// TODO: check that the version is at least 3.3 and throw error otherwise!console.log(`${PROXY_CHAIN.LOG_PREFIX}Checking Squid installation with '${PROXY_CHAIN.SQUID_CMD}@@ -92,7 +92,7 @@ export class ProxyChain {/*** Creates the temporary directory, writes a config to it and starts Squid process.- * @return Promise+ * @returns Promise* @private*/start() {@@ -195,7 +195,7 @@ export class ProxyChain {/*** Determines whether the Squid process is still running.- * @return Boolean+ * @returns Boolean* @private*/_isSquidRunning() {@@ -210,7 +210,7 @@ export class ProxyChain {/*** Generates Squid proxy configuration file and writes it to the temporary directory.- * @return Promise Promise resolving to the path to the configuration file.+ * @returns Promise Promise resolving to the path to the configuration file.* @private*/_writeSquidConf() {@@ -256,7 +256,7 @@ cache_peer_access my_peer allow all/*** Removes all proxy chains and terminates the squid process.- * @return {*}+ * @returns {*}*/shutdown() {const isRunning = this._isSquidRunning();diff --git a/src/utils.js b/src/utils.jsindex ff095c7e0..aab054326 100644--- a/src/utils.js+++ b/src/utils.js@@ -35,7 +35,7 @@ export const getPromisesDependency = () => {/*** Gets a promise dependency set using `setPromisesDependency()`,* or returns the native `Promise` function, or throws if no native promises are available.- * @return Promise+ * @returns Promise* @ignore*/export const getPromisePrototype = () => {@@ -52,7 +52,7 @@ export const getPromisePrototype = () => {/*** Returns a result of `Promise.resolve()` using promise library set by `setPromisesDependency()`,* or using native promises, or throws if no native promises are available.- * @return {*}+ * @returns {*}* @ignore*/export const newPromise = () => {@@ -73,7 +73,7 @@ export const nodeifyPromise = (promise, callback) => {/*** Creates an instance of ApifyClient using options as defined in the environment variables.* This function is exported in order to enable unit testing.- * @return {*}+ * @returns {*}* @ignore*/export const newClient = () => {@@ -129,7 +129,7 @@ export const parseUrl = (url) => {* to 'https://username:<redacted>@www.example.com/path#hash'* @param url URL, it must contain at least protocol and hostname* @param passwordReplacement The string that replaces password, by default it is '<redacted>'- * @return {string}+ * @returns {string}* @ignore*/export const redactUrl = (url, passwordReplacement) => {@@ -153,7 +153,7 @@ export const redactParsedUrl = (parsedUrl, passwordReplacement = '<redacted>') =* Adds charset=utf-8 to given content type if this parameter is missing.** @param contentType- * @return {string}+ * @returns {string}* @ignore*/export const addCharsetToContentType = (contentType) => {diff --git a/test/actor.js b/test/actor.jsindex 7d40a4729..1d063a061 100644--- a/test/actor.js+++ b/test/actor.js@@ -129,7 +129,7 @@ const testMain = (method, bodyRaw, contentType, userFunc, expectedExitCode = 0)/*** Helper function that enables testing of Apify.main()- * @return Promise+ * @returns Promise*/const testMain = ({ userFunc, exitCode }) => {// Mock process.exit() to check exit code and prevent process exit@@ -196,6 +196,7 @@ const getEmptyEnv = () => {startedAt: null,timeoutAt: null,defaultKeyValueStoreId: null,+ memoryMbytes: null,};};@@ -217,6 +218,7 @@ const setEnv = (env) => {if (env.startedAt) process.env.APIFY_STARTED_AT = env.startedAt.toISOString();if (env.timeoutAt) process.env.APIFY_TIMEOUT_AT = env.timeoutAt.toISOString();if (env.defaultKeyValueStoreId) process.env.APIFY_DEFAULT_KEY_VALUE_STORE_ID = env.defaultKeyValueStoreId;+ if (env.memoryMbytes) process.env.APIFY_MEMORY_MBYTES = env.memoryMbytes.toString();};describe('Apify.getEnv()', () => {@@ -238,6 +240,7 @@ describe('Apify.getEnv()', () => {startedAt: new Date('2017-01-01'),timeoutAt: new Date(),defaultKeyValueStoreId: 'some store',+ memoryMbytes: 1234,});setEnv(expectedEnv);