Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Added Apify.utils namespace and sleep() function (#87)

Jan Čurn898354615a95
diff --git a/package.json b/package.json
index 46f84a619..94f885dd6 100644
--- a/package.json
+++ b/package.json
@@ -49,6 +49,7 @@
"bluebird": "^3.5.0",
"content-type": "^1.0.3",
"fs-extra": "^5.0.0",
+ "jquery": "^3.3.1",
"pidusage": "^2.0.6",
"proxy-chain": "^0.1.26",
"ps-tree": "^1.1.0",
diff --git a/src/index.js b/src/index.js
index 86c07e197..e48808c95 100644
--- a/src/index.js
+++ b/src/index.js
@@ -13,8 +13,9 @@ import Request from './request';
import RequestList from './request_list';
import { openRequestQueue } from './request_queue';
import SettingsRotator from './settings_rotator';
-import { apifyClient, getMemoryInfo, isProduction, isAtHome } from './utils';
+import { apifyClient, getMemoryInfo, isProduction, isAtHome, publicUtils } from './utils';
import { browse, launchWebDriver } from './webdriver';
+import { puppeteerUtils } from './puppeteer_utils';
/* globals module */
@@ -83,4 +84,9 @@ module.exports = {
// Webdriver
browse,
launchWebDriver,
+
+ // utils
+ utils: Object.assign(publicUtils, {
+ puppeteer: puppeteerUtils,
+ }),
};
diff --git a/src/puppeteer_utils.js b/src/puppeteer_utils.js
new file mode 100644
index 000000000..f0770598f
--- /dev/null
+++ b/src/puppeteer_utils.js
@@ -0,0 +1,118 @@
+import fs from 'fs';
+import path from 'path';
+import Promise from 'bluebird';
+
+const readFilePromised = Promise.promisify(fs.readFile);
+
+/**
+ * Hides certain Puppeteer fingerprints from the page, in order to help avoid detection of the crawler.
+ * The function should be called on a newly-created page object before navigating to the target crawled page.
+ *
+ * @param page Puppeteer [Page](https://github.com/GoogleChrome/puppeteer/blob/master/docs/api.md#class-page) object.
+ * @return {Promise}
+ * @memberof utils.puppeteer
+ */
+const hideWebDriver = async (page) => {
+ await page.evaluateOnNewDocument(() => {
+ var modifiedNavigator; // eslint-disable-line no-var
+ try {
+ if (Navigator.prototype.hasOwnProperty('webdriver')) { // eslint-disable-line no-prototype-builtins
+ modifiedNavigator = Navigator.prototype;
+ } else {
+ modifiedNavigator = Object.create(window.navigator);
+ Object.defineProperty(window, 'navigator', {
+ value: modifiedNavigator,
+ configurable: false,
+ enumerable: true,
+ writable: false,
+ });
+ }
+ Object.defineProperties(modifiedNavigator, {
+ webdriver: {
+ configurable: true,
+ get: function () { // eslint-disable-line object-shorthand
+ return false;
+ },
+ },
+ });
+ // Date.prototype.getTimezoneOffset = function () { return -4 * 60; };
+ } catch (e) {
+ console.error(e);
+ }
+ });
+};
+
+
+/**
+ * Injects a JavaScript file into a Puppeteer page.
+ * Unlike Puppeteer's `addScriptTag` function, this function works on pages
+ * with arbitrary Cross-Origin Resource Sharing (CORS) policies.
+ *
+ * @param page Puppeteer [Page](https://github.com/GoogleChrome/puppeteer/blob/master/docs/api.md#class-page) object.
+ * @param filePath File path
+ * @return {Promise}
+ */
+const injectFile = async (page, filePath) => {
+ const contents = await readFilePromised(filePath, 'utf8');
+
+ return page.evaluate(contents);
+};
+
+
+/**
+ * Injects [jQuery](https://jquery.com/) library into a Puppeteer page.
+ * jQuery is often useful for various web scraping and crawling tasks,
+ * e.g. to extract data from HTML elements using CSS selectors.
+ *
+ * Beware that the injected jQuery object will be set to the `window.$` variable and thus it might cause conflicts with
+ * libraries included by the page that use the same variable (e.g. another version of jQuery).
+ *
+ * @param page Puppeteer [Page](https://github.com/GoogleChrome/puppeteer/blob/master/docs/api.md#class-page) object.
+ * @return {Promise}
+ * @memberof utils.puppeteer
+ */
+const injectJQuery = (page) => {
+ const scriptPath = path.resolve(path.join(__dirname, '../node_modules/jquery/dist/jquery.min.js'));
+ return injectFile(page, scriptPath);
+};
+
+
+/**
+ * Injects [Underscore.js](https://underscorejs.org/) library into a Puppeteer page.
+ * Beware that the injected Underscore object will be set to the `window._` variable and thus it might cause conflicts with
+ * libraries included by the page that use the same variable.
+ *
+ * @param page Puppeteer [Page](https://github.com/GoogleChrome/puppeteer/blob/master/docs/api.md#class-page) object.
+ * @return {Promise}
+ * @memberof utils.puppeteer
+ */
+const injectUnderscore = (page) => {
+ const scriptPath = path.resolve(path.join(__dirname, '../node_modules/underscore/underscore-min.js'));
+ return injectFile(page, scriptPath);
+};
+
+
+/**
+ * A namespace that contains various Puppeteer utilities.
+ *
+ * Example usage:
+ * ```javascript
+ * const Apify = require('apify');
+ *
+ * // Open https://www.example.com in Puppeteer
+ * const browser = await Apify.launchPuppeteer();
+ * const page = await browser.newPage();
+ * await page.goto('https://www.example.com');
+ *
+ * // Inject jQuery into a page
+ * await Apify.utils.puppeteer.injectJQueryScript(page);
+ * ```
+ * @namespace utils.puppeteer
+ * @name utils.puppeteer
+ */
+export const puppeteerUtils = {
+ hideWebDriver,
+ injectFile,
+ injectJQuery,
+ injectUnderscore,
+};
diff --git a/src/utils.js b/src/utils.js
index 78d8db9f4..7b540bc16 100644
--- a/src/utils.js
+++ b/src/utils.js
@@ -7,6 +7,7 @@ import ApifyClient from 'apify-client';
import psTree from 'ps-tree';
import pidusage from 'pidusage';
import _ from 'underscore';
+import { delayPromise } from 'apify-shared/utilities';
import { ENV_VARS } from './constants';
export const PID_USAGE_NOT_FOUND_ERROR = 'No maching pid found';
@@ -290,3 +291,44 @@ export const getTypicalChromeExecutablePath = () => {
* @function
*/
export const isAtHome = () => !!process.env[ENV_VARS.IS_AT_HOME];
+
+
+/**
+ * Returns a promise that resolves after a specific period of time. This is useful to implement waiting
+ * in your code, e.g. to prevent overloading of target website or to avoid bot detection.
+ *
+ * Example usage:
+ * ```javascript
+ * const Apify = require('apify);
+ *
+ * ...
+ *
+ * // Sleep 1.5 seconds
+ * await Apify.utils.sleep(1500);
+ * ```
+ * @param millis Period of time to sleep, in milliseconds. If not a positive number, the returned promise resolves immediately.
+ * @memberof utils
+ * @return {Promise}
+ */
+const sleep = (millis) => {
+ return delayPromise(millis);
+};
+
+
+/**
+ * A namespace that contains various utilities.
+ *
+ * Example usage:
+ * ```javascript
+ * const Apify = require('apify);
+ *
+ * ...
+ *
+ * // Sleep 1.5 seconds
+ * await Apify.utils.sleep(1500);
+ * ```
+ * @namespace utils
+ */
+export const publicUtils = {
+ sleep,
+};
diff --git a/test/events.js b/test/events.js
index 8c082326d..6e6f17584 100644
--- a/test/events.js
+++ b/test/events.js
@@ -117,6 +117,7 @@ describe('Apify.events', () => {
delete process.env[ENV_VARS.ACTOR_EVENTS_WS_URL];
});
+ // TODO: This tests fails if system is overloaded, it should not depend on timing!
it('should send persist state events in regular interval', async () => {
process.env.APIFY_TEST_PERSIST_INTERVAL_MILLIS = 20;
diff --git a/test/puppeteer_utils.js b/test/puppeteer_utils.js
new file mode 100644
index 000000000..11203013a
--- /dev/null
+++ b/test/puppeteer_utils.js
@@ -0,0 +1,89 @@
+import chai, { expect } from 'chai';
+import chaiAsPromised from 'chai-as-promised';
+import Apify from '../build/index';
+
+chai.use(chaiAsPromised);
+
+/* global process, describe, it */
+
+describe('Apify.utils.puppeteer', () => {
+ it('injectJQuery()', async () => {
+ const browser = await Apify.launchPuppeteer({ headless: true });
+
+ try {
+ const page = await browser.newPage();
+ await page.goto('about:blank');
+
+ // NOTE: Chrome already defines window.$ as alias to document.querySelector(),
+ // (https://developers.google.com/web/tools/chrome-devtools/console/command-line-reference#queryselector)
+ const result1 = await page.evaluate(() => {
+ return {
+ isDefined: window.jQuery !== undefined,
+ };
+ });
+ expect(result1).to.eql({
+ isDefined: false,
+ });
+
+ await Apify.utils.puppeteer.injectJQuery(page);
+ const result2 = await page.evaluate(() => {
+ return {
+ isDefined: window.jQuery === window.$,
+ text: $('h1').text(),
+ };
+ });
+ expect(result2).to.eql({
+ isDefined: true,
+ text: '',
+ });
+ } finally {
+ browser.close();
+ }
+ });
+
+ it('injectUnderscore()', async () => {
+ const browser = await Apify.launchPuppeteer({ headless: true });
+
+ try {
+ const page = await browser.newPage();
+ await page.goto('about:blank');
+
+ const result1 = await page.evaluate(() => {
+ return { isDefined: window._ !== undefined };
+ });
+ expect(result1).to.eql({ isDefined: false });
+
+ await Apify.utils.puppeteer.injectUnderscore(page);
+ const result2 = await page.evaluate(() => {
+ return { isDefined: _.isEmpty({}) };
+ });
+ expect(result2).to.eql({ isDefined: true });
+ } finally {
+ browser.close();
+ }
+ });
+
+ it('hideWebDriver()', async () => {
+ const browser = await Apify.launchPuppeteer({ headless: true });
+
+ try {
+ const page = await browser.newPage();
+ await page.goto('about:blank');
+
+ // TODO: Jarda, please can you add unit test for this?
+ /*
+ const result1 = await page.evaluate(() => {
+ return { isDefined: window._ !== undefined };
+ });
+ expect(result1).to.eql({ isDefined: false });
+
+ await Apify.utils.puppeteer.hideWebDriver(page);
+ const result2 = await page.evaluate(() => {
+ return { isDefined: _.isEmpty({}) };
+ });
+ expect(result2).to.eql({ isDefined: true }); */
+ } finally {
+ browser.close();
+ }
+ });
+});
diff --git a/test/utils.js b/test/utils.js
index e710c0b92..970c3769f 100644
--- a/test/utils.js
+++ b/test/utils.js
@@ -301,3 +301,30 @@ describe('utils.avg()', () => {
expect(utils.avg([9])).to.be.eql(9);
});
});
+
+describe('Apify.utils.sleep()', () => {
+ it('works', () => {
+ let timeBefore;
+ return Promise.resolve()
+ .then(() => {
+ return Apify.utils.sleep(0);
+ })
+ .then(() => {
+ return Apify.utils.sleep();
+ })
+ .then(() => {
+ return Apify.utils.sleep(null);
+ })
+ .then(() => {
+ return Apify.utils.sleep(-1);
+ })
+ .then(() => {
+ timeBefore = Date.now();
+ return Apify.utils.sleep(100);
+ })
+ .then(() => {
+ const timeAfter = Date.now();
+ expect(timeAfter - timeBefore).to.be.gte(95);
+ });
+ });
+});