Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Implementing actor events (#35)

Marek Trunkátf59bad70a8a9
diff --git a/README.md b/README.md
index ce00423c2..946936803 100644
--- a/README.md
+++ b/README.md
@@ -1,4 +1,4 @@
-# apify: Apify Actor runtime for JavaScript
+# apify: Apify JavaScript SDK
<!-- Mirror this part to src/index.js -->
[![npm version](https://badge.fury.io/js/apify.svg)](http://badge.fury.io/js/apify)
@@ -20,17 +20,17 @@ most common use-cases are:
<ul>
<li>
If you need to process high volume of <strong>asynchronous tasks in parallel</strong> then take a
- look at <a href="https://www.apify.com/docs/sdk/apify-runtime-js/beta#AutoscaledPool">AutoscaledPool</a>. This class executes defined tasks in a pool
+ look at <a href="https://www.apify.com/docs/sdk/apify-runtime-js/latest#AutoscaledPool">AutoscaledPool</a>. This class executes defined tasks in a pool
which size is scaled based on available memory and CPU.
</li>
<li>
If you want to <strong>crawl</strong> a list of urls using for example <a href="https://www.npmjs.com/package/request" target="_blank">
- Request</a> package then import those url as a <a href="https://www.apify.com/docs/sdk/apify-runtime-js/beta#RequestList">RequestList</a> and then use
- <a href="https://www.apify.com/docs/sdk/apify-runtime-js/beta#BasicCrawler">BasicCrawler</a> to process them in a pool.
+ Request</a> package then import those url as a <a href="https://www.apify.com/docs/sdk/apify-runtime-js/latest#RequestList">RequestList</a> and then use
+ <a href="https://www.apify.com/docs/sdk/apify-runtime-js/latest#BasicCrawler">BasicCrawler</a> to process them in a pool.
</li>
<li>
If you want to crawl a list of urls but you need a real <strong>browser</strong>. Then use
- <a href="https://www.apify.com/docs/sdk/apify-runtime-js/beta#PuppeteerCrawler">PuppeteerCrawler</a> which helps you to process a <a href="https://www.apify.com/docs/sdk/apify-runtime-js/beta#RequestList">RequestList</a>
+ <a href="https://www.apify.com/docs/sdk/apify-runtime-js/latest#PuppeteerCrawler">PuppeteerCrawler</a> which helps you to process a <a href="https://www.apify.com/docs/sdk/apify-runtime-js/latest#RequestList">RequestList</a>
using <a href="https://github.com/GoogleChrome/puppeteer" target="_blank">Puppeteer</a> (headless Chrome browser).
</li>
</ul>
@@ -46,11 +46,11 @@ we have few helper classes and functions:
`Apify.launchPuppeteer()` function starts new instance of Puppeteer browser and returns its browser object.
</li>
<li>
- <a href="https://www.apify.com/docs/sdk/apify-runtime-js/beta#PuppeteerPool">PuppeteerPool</a> helps to mantain a pool of Puppeteer instances. This is usefull
+ <a href="https://www.apify.com/docs/sdk/apify-runtime-js/latest#PuppeteerPool">PuppeteerPool</a> helps to mantain a pool of Puppeteer instances. This is usefull
when you need to restart browser after certain number of requests to rotate proxy servers.
</li>
<li>
- <a href="https://www.apify.com/docs/sdk/apify-runtime-js/beta#PuppeteerCrawler">PuppeteerCrawler</a> helps to crawl a <a href="https://www.apify.com/docs/sdk/apify-runtime-js/beta#RequestList">RequestList</a>
+ <a href="https://www.apify.com/docs/sdk/apify-runtime-js/latest#PuppeteerCrawler">PuppeteerCrawler</a> helps to crawl a <a href="https://www.apify.com/docs/sdk/apify-runtime-js/latest#RequestList">RequestList</a>
in a autoscaled pool.
</li>
</ul>
diff --git a/package.json b/package.json
index 441377aff..f5baadbd5 100644
--- a/package.json
+++ b/package.json
@@ -1,7 +1,7 @@
{
"name": "apify",
"version": "0.5.6",
- "description": "Apify Actor runtime for JavaScript",
+ "description": "Apify JavaScript SDK",
"main": "build/index.js",
"keywords": [
"apify",
@@ -50,7 +50,8 @@
"fs-extra": "^5.0.0",
"proxy-chain": "^0.1.2",
"request-promise": "^4.2.2",
- "underscore": "^1.8.3"
+ "underscore": "^1.8.3",
+ "ws": "^4.1.0"
},
"devDependencies": {
"apify-jsdoc-template": "github:apifytech/apify-jsdoc-template",
diff --git a/src/actor.js b/src/actor.js
index 51483ae79..add05f79d 100644
--- a/src/actor.js
+++ b/src/actor.js
@@ -1,6 +1,9 @@
import _ from 'underscore';
import Promise from 'bluebird';
import fs from 'fs';
+import EventEmitter from 'events';
+import WebSocket from 'ws';
+import log from 'apify-shared/log';
import { checkParamOrThrow } from 'apify-client/build/utils';
import { ENV_VARS, EXIT_CODES, ACT_TASK_TERMINAL_STATUSES, DEFAULT_PROXY_HOSTNAME, DEFAULT_PROXY_PORT } from './constants';
import { newPromise, apifyClient, addCharsetToContentType } from './utils';
@@ -19,6 +22,75 @@ const tryParseDate = (str) => {
return unix > 0 ? new Date(unix) : undefined;
};
+/**
+ * Event emitter providing access to events from Actor infrastructure. Event emitter is initiated by Apify.main().
+ * If you don't use Apify.main() then you must call `await Apify.initializeEvents()` yourself.
+ *
+ * Example usage:
+ *
+ * ```javascript
+ * import { ACTOR_EVENT_NAMES } from 'apify/constants';
+ *
+ * Apify.main(async () => {
+ * &nbsp;
+ * Apify.events.on(ACTOR_EVENT_NAMES.CPU_INFO, (data) => {
+ * if (data.isCpuOverloaded) console.log('OH NO! We are overloading CPU!');
+ * });
+ *. &nbsp;
+ * });
+ * ```
+ *
+ * See <a href="https://nodejs.org/api/events.html#events_class_eventemitter" target="_blank">NodeJs documentation</a>
+ * for more information about event emitter use.
+ *
+ * @memberof module:Apify
+ * @name events
+ */
+export const events = new EventEmitter();
+
+/**
+ * Websocket connection to actor events.
+ * @ignore
+ */
+let eventsWs = null;
+
+/**
+ * Initializes Apify.events event emitter by creating connection to a websocket that provides them.
+ *
+ * @memberof module:Apify
+ * @name initializeEvents
+ * @function
+ */
+export const initializeEvents = () => {
+ if (eventsWs) return;
+
+ const eventsWsUrl = process.env[ENV_VARS.ACTOR_EVENTS_WS_URL];
+
+ // Localy there is no web socket to connect so just print a warning.
+ if (!eventsWsUrl) {
+ log.warning(`Environment variable ${ENV_VARS.ACTOR_EVENTS_WS_URL} was not provided. Events in Apify.events emitter won't be available!`);
+ return;
+ }
+
+ eventsWs = new WebSocket(eventsWsUrl);
+ eventsWs.on('message', (message) => {
+ if (!message) return;
+
+ try {
+ const { name, data } = JSON.parse(message);
+
+ events.emit(name, data);
+ } catch (err) {
+ log.exception(err, 'Cannot parse actor event');
+ }
+ });
+ eventsWs.on('error', err => log.exception(err, 'Actor events web socket connection failed'));
+ eventsWs.on('close', () => {
+ log.warning('Actor events websocket has been closed');
+ eventsWs = null;
+ });
+};
+
/**
* Returns a new object which contains information parsed from the `APIFY_XXX` environment variables.
* It has the following properties:
@@ -162,9 +234,8 @@ export const main = (userFunc) => {
try {
newPromise()
- .then(() => {
- return userFunc();
- })
+ .then(() => initializeEvents())
+ .then(() => userFunc())
.catch((err) => {
clearInterval(intervalId);
if (!exited) {
diff --git a/src/autoscaled_pool.js b/src/autoscaled_pool.js
index 9ab556d85..459a9919a 100644
--- a/src/autoscaled_pool.js
+++ b/src/autoscaled_pool.js
@@ -3,6 +3,8 @@ import log from 'apify-shared/log';
import _ from 'underscore';
import { checkParamOrThrow } from 'apify-client/build/utils';
import { getMemoryInfo, isPromise } from './utils';
+import { events } from './actor';
+import { ACTOR_EVENT_NAMES } from './constants';
const MEM_CHECK_INTERVAL_MILLIS = 200; // This is low to have at least.
const MAYBE_RUN_INTERVAL_MILLIS = 1000;
@@ -28,7 +30,7 @@ const humanReadable = bytes => `${Math.round(bytes / 1024 / 1024)} MB`;
/**
* AutoscaledPool helps to process asynchronous task in parallel. It scales the number of concurrent tasks based on
- * the available memory. If any of the tasks throws an error then the pool.run() method
+ * the available memory and CPU. If any of the tasks throws an error then the pool.run() method
* also throws.
*
* Basic usage of AutoscaledPool:
@@ -76,6 +78,7 @@ export default class AutoscaledPool {
this.runningPromises = {};
this.runningCount = 0;
this.freeBytesSnapshots = [];
+ this.isCpuOverloaded = false;
// Intervals.
this.memCheckInterval = null;
@@ -85,6 +88,12 @@ export default class AutoscaledPool {
// which gets resolved once everything is done.
this.resolve = null;
this.reject = null;
+
+ // Connect to actor events for CPU info.
+ this.cpuInfoListener = (data) => {
+ this.isCpuOverloaded = data.isCpuOverloaded;
+ };
+ events.on(ACTOR_EVENT_NAMES.CPU_INFO, this.cpuInfoListener);
}
/**
@@ -139,8 +148,8 @@ export default class AutoscaledPool {
// Maybe scale down.
if (
this.intervalCounter % SCALE_DOWN_INTERVAL === 0
- && (freeBytes / totalBytes < this.minFreeMemoryRatio)
&& this.concurrency > this.minConcurrency
+ && (this.isCpuOverloaded || freeBytes / totalBytes < this.minFreeMemoryRatio)
) {
this.concurrency--;
log.debug('AutoscaledPool: scaling down', { concurrency: this.concurrency });
@@ -266,6 +275,8 @@ export default class AutoscaledPool {
this.resolve = null;
this.reject = null;
+ events.removeListener(ACTOR_EVENT_NAMES.CPU_INFO, this.cpuInfoListener);
+
clearInterval(this.memCheckInterval);
clearInterval(this.maybeRunInterval);
}
diff --git a/src/constants.js b/src/constants.js
index a737f39bf..e42550264 100644
--- a/src/constants.js
+++ b/src/constants.js
@@ -4,7 +4,6 @@ consts.DEFAULT_USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) App
consts.DEFAULT_PROXY_HOSTNAME = 'proxy.apify.com';
consts.DEFAULT_PROXY_PORT = 8000;
-
/**
* Exit codes for the act process.
* The error codes must be in range 1-128, to avoid collision with signal exits
@@ -38,6 +37,7 @@ consts.ENV_VARS = {
INTERNAL_PORT: 'APIFY_INTERNAL_PORT',
MEMORY_MBYTES: 'APIFY_MEMORY_MBYTES',
LOG_LEVEL: 'APIFY_LOG_LEVEL',
+ ACTOR_EVENTS_WS_URL: 'APIFY_ACTOR_EVENTS_WS_URL',
};
consts.KEY_VALUE_STORE_KEYS = {
diff --git a/src/index.js b/src/index.js
index 7d760ea52..65c844077 100644
--- a/src/index.js
+++ b/src/index.js
@@ -1,8 +1,6 @@
-import EventEmitter from 'events';
import log from 'apify-shared/log';
-
import { ENV_VARS } from './constants';
-import { main, readyFreddy, getEnv, call, getApifyProxyUrl } from './actor';
+import { main, readyFreddy, getEnv, call, getApifyProxyUrl, events, initializeEvents } from './actor';
import AutoscaledPool from './autoscaled_pool';
import BasicCrawler from './basic_crawler';
import { pushData, openDataset } from './dataset';
@@ -30,8 +28,8 @@ if (!isProduction() || process.env[ENV_VARS.LOG_LEVEL] === 'DEBUG') log.isDebugM
*
* For more information about the Apify Actor platform, please go to
* {@link https://www.apify.com/docs/actor|Actor documentation}.
-
- * The source code of this package is available on {@link https://github.com/apifytech/apify-runtime-js|GitHub}.
+ *
+ * The source code of this package is available on {@link https://github.com/apifytech/apify-js|GitHub}.
*
* <h2>Example usage</h2>
*
@@ -107,6 +105,47 @@ if (!isProduction() || process.env[ENV_VARS.LOG_LEVEL] === 'DEBUG') log.isDebugM
* npm install apify --save
* ```
*
+ * <h2>Local usage</h2>
+ *
+ * You can use Apify package locally. To do that you must define following environment variables
+ *
+ * <table class="table table-bordered table-condensed">
+ * <thead>
+ * <tr>
+ * <th>Environment variable</th>
+ * <th>Description</th>
+ * </tr>
+ * </thead>
+ * <tbody>
+ * <tr>
+ * <td><code>APIFY_LOCAL_EMULATION_DIR</code></td>
+ * <td>
+ * Directory where apify package locally emulates Apify storages - key-value store and dataset.
+ * Key-value stores will be emulated in directory
+ * <code>[APIFY_LOCAL_EMULATION_DIR]/key-value-stores/[STORE_ID]</code>
+ * and datasets in directory
+ * <code>[APIFY_LOCAL_EMULATION_DIR]/datasets/[DATESET_ID]</code>.
+ * </td>
+ * </tr>
+ * <tr>
+ * <td><code>APIFY_DEFAULT_KEY_VALUE_STORE_ID</code></td>
+ * <td>Store ID of default key-value store.</td>
+ * </tr>
+ * <tr>
+ * <td><code>APIFY_DEFAULT_DATASET_ID</code></td>
+ * <td>Dataset ID of default dataset.</td>
+ * </tr>
+ * </tbody>
+ * </table>
+ *
+ * Apify will then store key-value store records in files named <code>[KEY].[EXT]</code> where <code>[KEY]</code>
+ * is the record key and <code>[EXT]</code> is based on the record content type. Dataset items will be stored
+ * in files named <code>[ID].json</code> where <code>[ID]</code> is sequence number of your dataset item.
+ *
+ * If you want to use <a href="https://www.apify.com/docs/proxy" target="_blank">Apify Proxy</a> locally
+ * then you must define an environment variable <code>PROXY_PASSWORD</code> with password you find at
+ * <a href="https://my.apify.com/proxy" target="_blank">https://my.apify.com/proxy</a>.
+ *
* <h2>Promises vs. callbacks</h2>
*
* By default, all asynchronous functions provided by this package return a promise.
@@ -124,10 +163,10 @@ if (!isProduction() || process.env[ENV_VARS.LOG_LEVEL] === 'DEBUG') log.isDebugM
* @module Apify
*/
module.exports = {
- events: new EventEmitter(),
-
// Actor
main,
+ events,
+ initializeEvents,
getEnv,
call,
readyFreddy,
diff --git a/test/actor.js b/test/actor.js
index d3eb19c0f..b5e08ae37 100644
--- a/test/actor.js
+++ b/test/actor.js
@@ -1,12 +1,17 @@
import fs from 'fs';
import _ from 'underscore';
-import { expect } from 'chai';
+import chai, { expect } from 'chai';
+import chaiAsPromised from 'chai-as-promised';
+import 'babel-polyfill';
+import WebSocket from 'ws';
import sinon from 'sinon';
import tmp from 'tmp';
import Promise from 'bluebird';
import { delayPromise } from 'apify-shared/utilities';
import { ACT_TASK_STATUSES, ENV_VARS, DEFAULT_PROXY_PORT, DEFAULT_PROXY_HOSTNAME } from '../build/constants';
+chai.use(chaiAsPromised);
+
// NOTE: test use of require() here because this is how its done in acts
const Apify = require('../build/index');
@@ -719,3 +724,73 @@ describe('Apify.getApifyProxyUrl()', () => {
})).to.be.eql('http://auto:[email protected]:345');
});
});
+
+describe('Apify.events', () => {
+ it('should work in Apify.main()', (done) => {
+ const wss = new WebSocket.Server({ port: 9099 });
+ const eventsReceived = [];
+
+ // Create server that sends events
+ wss.on('connection', (ws, req) => {
+ expect(req.url).to.be.eql('/someRunId');
+
+ const send = obj => ws.send(JSON.stringify(obj));
+
+ setTimeout(() => send({ name: 'name-1', data: [1, 2, 3] }), 50);
+ setTimeout(() => send({ name: 'name-1', data: { foo: 'bar' } }), 100);
+ setTimeout(() => send({ name: 'name-2', data: [1] }), 50);
+ setTimeout(() => send({ name: 'name-2', data: [2] }), 50);
+ });
+
+ process.env[ENV_VARS.ACTOR_EVENTS_WS_URL] = 'ws://localhost:9099/someRunId';
+
+ // Run main and store received events
+ Apify.main(async () => {
+ Apify.events.on('name-1', data => eventsReceived.push(data));
+ await delayPromise(1000);
+ });
+
+ // Main will call process.exit() so we must stub it.
+ const stubbedExit = sinon
+ .stub(process, 'exit')
+ .callsFake((code) => {
+ expect(code).to.be.eql(0);
+ expect(eventsReceived).to.be.eql([[1, 2, 3], { foo: 'bar' }]);
+
+ // Cleanup.
+ stubbedExit.restore();
+ wss.close();
+ delete process.env[ENV_VARS.ACTOR_EVENTS_WS_URL];
+ done();
+ });
+ });
+
+ it('should work without Apify.main()', async () => {
+ const wss = new WebSocket.Server({ port: 9099 });
+ const eventsReceived = [];
+
+ wss.on('connection', (ws, req) => {
+ expect(req.url).to.be.eql('/someRunId');
+
+ const send = obj => ws.send(JSON.stringify(obj));
+
+ setTimeout(() => send({ name: 'name-1', data: [1, 2, 3] }), 50);
+ setTimeout(() => send({ name: 'name-1', data: { foo: 'bar' } }), 100);
+ setTimeout(() => send({ name: 'name-2', data: [1] }), 50);
+ setTimeout(() => send({ name: 'name-2', data: [2] }), 50);
+ });
+
+ process.env[ENV_VARS.ACTOR_EVENTS_WS_URL] = 'ws://localhost:9099/someRunId';
+
+ // Connect to websocket and receive events.
+ await Apify.initializeEvents();
+ Apify.events.on('name-1', data => eventsReceived.push(data));
+ await delayPromise(1000);
+
+ expect(eventsReceived).to.be.eql([[1, 2, 3], { foo: 'bar' }]);
+
+ // Cleanup.
+ wss.close();
+ delete process.env[ENV_VARS.ACTOR_EVENTS_WS_URL];
+ });
+});
diff --git a/test/autoscaled_pool.js b/test/autoscaled_pool.js
index dbf856fac..b75d2fc6f 100644
--- a/test/autoscaled_pool.js
+++ b/test/autoscaled_pool.js
@@ -5,6 +5,7 @@ import _ from 'underscore';
import sinon from 'sinon';
import { delayPromise } from 'apify-shared/utilities';
import * as Apify from '../build/index';
+import { ACTOR_EVENT_NAMES } from '../build/constants';
import { LOG_INFO_INTERVAL, SCALE_UP_MAX_STEP } from '../build/autoscaled_pool';
import * as utils from '../build/utils';
@@ -160,5 +161,58 @@ describe('autoscaled_pool', () => {
await expect(pool.run()).to.be.rejectedWith('some-error');
});
+
+ it('should scale down when CPU is overloaded', async () => {
+ const pool = new Apify.AutoscaledPool({
+ minConcurrency: 1,
+ maxConcurrency: 100,
+ minFreeMemoryRatio: 0.1,
+ workerFunction: async () => {},
+ });
+ const mock = sinon.mock(utils);
+
+ // Should scale up.
+ pool.intervalCounter = LOG_INFO_INTERVAL - 1;
+ mock.expects('getMemoryInfo')
+ .once()
+ .returns(Promise.resolve({ freeBytes: toBytes(9.99), totalBytes: toBytes(10) }));
+ await pool._autoscale();
+ expect(pool.concurrency).to.be.eql(1 + SCALE_UP_MAX_STEP);
+
+ // Should scale up.
+ pool.intervalCounter = LOG_INFO_INTERVAL - 1;
+ mock.expects('getMemoryInfo')
+ .once()
+ .returns(Promise.resolve({ freeBytes: toBytes(9.99), totalBytes: toBytes(10) }));
+ await pool._autoscale();
+ expect(pool.concurrency).to.be.eql(1 + (2 * SCALE_UP_MAX_STEP));
+
+ // Emit CPU overloaded = true event.
+ Apify.events.emit(ACTOR_EVENT_NAMES.CPU_INFO, { isCpuOverloaded: true });
+
+ // Should scale up but because of CPU overloaded event it always scales down.
+ for (let i = 1; i <= 5; i++) {
+ pool.intervalCounter = LOG_INFO_INTERVAL - 1;
+ mock.expects('getMemoryInfo')
+ .once()
+ .returns(Promise.resolve({ freeBytes: toBytes(9.99), totalBytes: toBytes(10) }));
+ await pool._autoscale(); //eslint-disable-line
+ expect(pool.concurrency).to.be.eql((1 + (2 * SCALE_UP_MAX_STEP)) - i);
+ }
+
+ // Emit CPU overloaded = false event.
+ Apify.events.emit(ACTOR_EVENT_NAMES.CPU_INFO, { isCpuOverloaded: false });
+
+ // Should scale up again.
+ pool.intervalCounter = LOG_INFO_INTERVAL - 1;
+ mock.expects('getMemoryInfo')
+ .once()
+ .returns(Promise.resolve({ freeBytes: toBytes(9.99), totalBytes: toBytes(10) }));
+ await pool._autoscale();
+ expect(pool.concurrency).to.be.eql((1 + (3 * SCALE_UP_MAX_STEP)) - 5);
+
+ mock.verify();
+ mock.restore();
+ });
});