Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
init generating docs with jsdoc
JakubDrobnikdfc7ce7196b0
diff --git a/.gitignore b/.gitignoreindex 6e3dcd056..5fa5425a1 100644--- a/.gitignore+++ b/.gitignore@@ -9,4 +9,5 @@ coveragelogspids.idea-yarn.lock\ No newline at end of file+yarn.lock+docs\ No newline at end of filediff --git a/jsdoc/conf.json b/jsdoc/conf.jsonnew file mode 100644index 000000000..a3f7b3939--- /dev/null+++ b/jsdoc/conf.json@@ -0,0 +1,27 @@+{+ "tags": {+ "allowUnknownTags": true+ },+ "source": {+ "include": [ "build" ],+ "exclude": [ "build/constants.js", "build/proxy_chain.js" ],+ "includePattern": ".+\\.js(doc|x)?$",+ "excludePattern": "(^|\\/|\\\\)_"+ },+ "plugins": [+ "./node_modules/jsdoc-export-default-interop/dist/index.js",+ "plugins/markdown"+ ],+ "templates": {+ "cleverLinks": false,+ "monospaceLinks": false,+ "default": {+ "outputSourceFiles": false+ }+ },+ "opts": {+ "template": "./node_modules/apify-jsdoc-template",+ "encoding": "utf8",+ "destination": ".docs"+ }+}diff --git a/package.json b/package.jsonindex 38e8f5b88..750530bfb 100644--- a/package.json+++ b/package.json@@ -32,6 +32,7 @@],"scripts": {"build": "babel src --out-dir build",+ "build-doc": "npm run clean && npm run build && node ./node_modules/jsdoc/jsdoc.js --package ./package.json -c ./jsdoc/conf.json -d docs","test": "npm run build && mocha --compilers js:babel-core/register --recursive","test-cov": "npm run build && babel-node node_modules/isparta/bin/isparta cover --report html --report text node_modules/.bin/_mocha","prepare": "npm run build",@@ -68,7 +69,10 @@"proxy": "^0.2.4","sinon": "^2.2.0","sinon-stub-promise": "^4.0.0",- "request": "^2.81.0"+ "request": "^2.81.0",+ "jsdoc": "^3.4.3",+ "jsdoc-export-default-interop": "^0.3.1",+ "apify-jsdoc-template": "github:Apifier/apify-jsdoc-template"},"optionalDependencies": {"selenium-webdriver": "^3.4.0"diff --git a/publish.sh b/publish.shindex 7e2164694..d5f10171f 100755--- a/publish.sh+++ b/publish.sh@@ -11,11 +11,21 @@ BRANCH=`git status | grep 'On branch' | cut -d ' ' -f 3`BRANCH_UP_TO_DATE=`git status | grep 'nothing to commit' | tr -s \n ' '`;GIT_TAG="v${PACKAGE_VERSION}"+# Upload doc to S3 configuration+DOC_DIR=${PWD}"/docs"+AWS_BUCKET="apify-runtime-js-doc"+if [ -z "${BRANCH_UP_TO_DATE}" ]; thenprintf "${RED}You have uncommitted changes!${NC}\n"exit 1fi+echo "Generating documentation ..."+npm run build-doc++echo "Uploading docs to S3 ..."+aws s3 cp "${DOC_DIR}/" "s3://${AWS_BUCKET}/${GIT_TAG}/" --recursive --region us-east-1 --acl public-read --cache-control "public, max-age=86400"+echo "Pushing to git ..."git push@@ -28,6 +38,8 @@ if [ "${BRANCH}" = "master" ]; thenelseecho "Tagging version ${PACKAGE_VERSION} with tag \"latest\" ..."RUNNING_FROM_SCRIPT=1 npm dist-tag add ${PACKAGE_NAME}@${PACKAGE_VERSION} latest+ echo "Copy doc to latest folder..."+ aws s3 cp "s3://${AWS_BUCKET}/${GIT_TAG}/" "s3://${AWS_BUCKET}/latest/" --recursive --region us-east-1 --acl public-read --cache-control "public, max-age=86400"fi# Develop branch gets published as BETA and we don't allow to override tag of existing version.@@ -40,6 +52,9 @@ elif [ "${BRANCH}" = "develop" ]; thengit push origin ${GIT_TAG}echo "Git tag: ${GIT_TAG} created."+ echo "Copy docs to S3 to beta folder..."+ aws s3 cp "s3://${AWS_BUCKET}/${GIT_TAG}/" "s3://${AWS_BUCKET}/beta/" --recursive --region us-east-1 --acl public-read --cache-control "public, max-age=86400"+# For other branch throw an error.elseprintf "${RED}You can publish from develop and master branches only!${NC}\n"diff --git a/src/actor.js b/src/actor.jsindex 81573f5e4..33ccee9c0 100644--- a/src/actor.js+++ b/src/actor.js@@ -11,7 +11,10 @@ import { getPromisePrototype, newPromise, nodeifyPromise, newClient, addCharsetT/* global process, Buffer *//**- * A default instance of ApifyClient class.+ * @type ApifyClient+ * @memberof module:Apify+ * @function client+ * @description A default instance of ApifyClient class.* It can be configured by calling setOptions() function.*/export const apifyClient = newClient();@@ -25,6 +28,7 @@ const statPromised = Promise.promisify(fs.stat);* Tries to parse a string with date.* @param str Date string* @return Returns either a Date object or undefined+ * @ignore*/const tryParseDate = (str) => {const unix = Date.parse(str);@@ -39,7 +43,9 @@ const getDefaultStoreIdOrThrow = () => {/**- * Gets a value from the default key-value store for the current act run.+ * @memberof module:Apify+ * @function+ * @description Gets a value from the default key-value store for the current act run.* This store is created automatically for this run,* whose ID is defined in the `APIFY_DEFAULT_KEY_VALUE_STORE_ID` environment variable.* The result of the function is the body of the record. For records with 'application/json'@@ -127,7 +133,9 @@ export const getValue = (key, callback = null) => {};/**- * Stores a value in the default key-value store for the current act run.+ * @memberof module:Apify+ * @function+ * @description Stores a value in the default key-value store for the current act run.* This data is stored in the key-value store created specifically for this run,* whose ID is defined in the `APIFY_DEFAULT_KEY_VALUE_STORE_ID` environment variable.* The function has no result, but throws on invalid args or other errors.@@ -140,7 +148,7 @@ export const getValue = (key, callback = null) => {* If no contentType is specified, the value can be any object and it will be stringified to JSON.* If contentType is specified, value is considered raw data and it must be a String or Buffer.* For any other value an error will be thrown.- * @param options Optional settings, currently only { contentType: String } is supported to set MIME content type of the value.+ * @param options Optional settings, currently only \{ contentType: String \} is supported to set MIME content type of the value.* @param callback Optional callback.* @return Returns a promise if no callback was provided, otherwise the return value is not defined.*/@@ -238,7 +246,9 @@ export const setValue = (key, value, options, callback = null) => {/**- * Generates an object which contains parsed environment variables:+ * @memberof module:Apify+ * @function+ * @description Generates an object which contains parsed environment variables:* ```javascript* {* actId: String,@@ -270,7 +280,48 @@ export const getEnv = () => {};};-+/**+ * @memberof module:Apify+ * @function+ * @description To simplify development of acts, the runtime provides the `Apify.main(func)` function which does the following:+ * 1) Invokes the user function `func`+ * 2) If the function returned a promise, waits for it to resolve+ * 3) Exits the process+ * If the user function throws an exception or some other error is encountered,+ * then `Apify.main()` prints the details to console so that they are stored to the log file.+ * `Apify.main()` accepts a single argument - the user function that performs the operation of the act.+ * In the simplest case, the user function is synchronous:+ * ```javascript+ * Apify.main(() => {+ * // my synchronous function that returns immediately+ * });+ * ```+ * If the user function returns a promise, it is considered as asynchronous:+ * ```javascript+ * const request = require('request-promise');+ * Apify.main(() => {+ * // my asynchronous function that returns a promise+ * return Promise.resolve()+ * .then(() => {+ * return request('http://www.example.com');+ * })+ * .then((html) => {+ * console.log(html);+ * });+ * });+ * ```+ * To simplify your code, you can take advantage of the `async`/`await` keywords:+ * ```javascript+ * const request = require('request-promise');+ * Apify.main(async () => {+ * const html = await request('http://www.example.com');+ * console.log(html);+ * });+ * ```+ * Note that the `Apify.main()` function does not need to be used at all,+ * it is provided merely for user convenience.+ * @param userFunc+ */export const main = (userFunc) => {if (!userFunc || typeof (userFunc) !== 'function') {throw new Error('Handler function must be provided as a parameter');@@ -318,7 +369,9 @@ export const main = (userFunc) => {// TODO: this should rather be called Apify.listeningOnPort() or something like that/**- * Notifies Apify runtime that act is listening on port specified by the APIFY_INTERNAL_PORT environment+ * @memberof module:Apify+ * @function+ * @description Notifies Apify runtime that act is listening on port specified by the APIFY_INTERNAL_PORT environment* variable and is ready to receive a HTTP request with act input.*/export const readyFreddy = () => {@@ -333,8 +386,9 @@ export const readyFreddy = () => {};/**- * Executes given, waits for it to finish and fetches it's OUTPUT from key-value store and saves it to run.output.- *+ * @memberof module:Apify+ * @function+ * @description Executes given, waits for it to finish and fetches it's OUTPUT from key-value store and saves it to run.output.* @param {Object} opts* @param {String} opts.actId - Either act ID or username/actname.* @param {String} [opts.token] - User API token.diff --git a/src/browser.js b/src/browser.jsindex 16c780b82..4e03b5200 100644--- a/src/browser.js+++ b/src/browser.js@@ -20,6 +20,7 @@ import { ProxyChain } from './proxy_chain';/*** Gets the default options for the browse() function, generated from current process environment* variables. This is function to enable unit testing.+ * @ignore*/export const getDefaultBrowseOptions = () => {return {@@ -180,6 +181,7 @@ export class Browser {* @param url Optional string* @param options Optional object* @param callback Optional function+ * @ignore*/export const processBrowseArgs = (url, options, callback) => {if (typeof (url) === 'object' || typeof (url) === 'function') {@@ -203,10 +205,12 @@ export const processBrowseArgs = (url, options, callback) => {/**- * Opens a new web browser, which is attached to Apify debugger so that snapshots are sent to Run console (TODO).+ * @description Opens a new web browser, which is attached to Apify debugger so that snapshots are sent to Run console (TODO).* Internally, this function calls Selenium WebDrivers's Builder command to create a new WebDriver instance.* (see http://seleniumhq.github.io/selenium/docs/api/javascript/module/selenium-webdriver/index_exports_Builder.html)* The result of the function is a new instance of the Browser class.+ * @memberof module:Apify+ * @function* @param url Optional start URL to open. Defaults to about:blank* @param options Optional settings, their defaults are provided by the getDefaultBrowseOptions() function.* @param callback Optional callback.diff --git a/src/index.js b/src/index.jsindex 6724bbf18..18869fbee 100644--- a/src/index.js+++ b/src/index.js@@ -21,6 +21,16 @@ const Apify = {events: new EventEmitter(),};-// export this way so that we can import using:-// const Apify = require('apify');+/**+ * Helper package that simplifies development of Apify acts.+ * @module Apify+ * @description Basic usage of Apify module in Apify acts:+ * ```javascript+ * const Apify = require('apify');+ *+ * Apify.main(() => {+ * // my synchronous function that returns immediately+ * });+ * ```+ */module.exports = Apify;diff --git a/src/utils.js b/src/utils.jsindex 2decd9a04..3d877b09a 100644--- a/src/utils.js+++ b/src/utils.js@@ -12,7 +12,9 @@ let PromisesDependency = typeof Promise === 'function' ? Promise : null;// TODO: use bluebird promise by default instead of native promise, rename newPromise() to newUserPromise(), get rid of nodeifyPromise()/**- * Sets the promise dependency the SDK will use wherever Promises are returned.+ * @memberof module:Apify+ * @function+ * @description Sets the promise dependency the SDK will use wherever Promises are returned.* Passing `null` will force the SDK to use native Promises if they are available.* @param [Constructor] dep A reference to a Promise constructor*/@@ -22,7 +24,9 @@ export const setPromisesDependency = (dep) => {};/**- * Gets the promise dependency set by `Apify.setPromisesDependency`.+ * @memberof module:Apify+ * @function+ * @description Gets the promise dependency set by `Apify.setPromisesDependency`.*/export const getPromisesDependency = () => {return PromisesDependency;@@ -32,6 +36,7 @@ export const getPromisesDependency = () => {* Gets a promise dependency set using `setPromisesDependency()`,* or returns the native `Promise` function, or throws if no native promises are available.* @return Promise+ * @ignore*/export const getPromisePrototype = () => {if (PromisesDependency) {@@ -48,6 +53,7 @@ export const getPromisePrototype = () => {* Returns a result of `Promise.resolve()` using promise library set by `setPromisesDependency()`,* or using native promises, or throws if no native promises are available.* @return {*}+ * @ignore*/export const newPromise = () => {return getPromisePrototype().resolve();@@ -68,6 +74,7 @@ export const nodeifyPromise = (promise, callback) => {* Creates an instance of ApifyClient using options as defined in the environment variables.* This function is exported in order to enable unit testing.* @return {*}+ * @ignore*/export const newClient = () => {const opts = {@@ -87,6 +94,7 @@ export const newClient = () => {/*** Sames are Node's url.parse() just adds the 'username', 'password' and 'scheme' fields.* @param url+ * @ignore*/export const parseUrl = (url) => {const parsed = urlModule.parse(url);@@ -122,6 +130,7 @@ export const parseUrl = (url) => {* @param url URL, it must contain at least protocol and hostname* @param passwordReplacement The string that replaces password, by default it is '<redacted>'* @return {string}+ * @ignore*/export const redactUrl = (url, passwordReplacement) => {return redactParsedUrl(parseUrl(url), passwordReplacement);@@ -145,6 +154,7 @@ export const redactParsedUrl = (parsedUrl, passwordReplacement = '<redacted>') =** @param contentType* @return {string}+ * @ignore*/export const addCharsetToContentType = (contentType) => {if (!contentType) return contentType;