Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Implementing Apify.call()

Marek Trunkat4459f487f59d
diff --git a/src/actor.js b/src/actor.js
index eb49d75d3..b02436dda 100644
--- a/src/actor.js
+++ b/src/actor.js
@@ -1,6 +1,8 @@
import fs from 'fs';
import _ from 'underscore';
-import { ENV_VARS, EXIT_CODES } from './constants';
+import { checkParamOrThrow } from 'apify-client/build/utils';
+import { MAX_WAIT_FOR_FINISH_SECS } from 'apify-client/build/acts';
+import { ENV_VARS, EXIT_CODES, ACT_TASK_TERMINAL_STATUSES, DEFAULT_APIFY_CALL_WAIT_SECS } from './constants';
import { getPromisePrototype, newPromise, nodeifyPromise, newClient } from './utils';
/* global process, Buffer */
@@ -259,3 +261,68 @@ export const readyFreddy = () => {
console.log(`WARNING: ${ENV_VARS.WATCH_FILE} environment variable not specified, readyFreddy() has no effect.`);
}
};
+
+export const call = (actId, opts = {}) => {
+ const { acts, keyValueStores } = apifyClient;
+
+ // Common options.
+ const { token } = opts;
+ checkParamOrThrow(actId, 'actId', 'String');
+ checkParamOrThrow(token, 'token', 'Maybe String');
+ const defaultOpts = { actId };
+ if (token) defaultOpts.token = token;
+
+ // RunAct() options.
+ const { build, body, contentType } = opts;
+ checkParamOrThrow(build, 'build', 'Maybe String');
+ checkParamOrThrow(body, 'body', 'Maybe Buffer | String');
+ checkParamOrThrow(contentType, 'contentType', 'Maybe String');
+ const runActOpts = {};
+ if (contentType) runActOpts.contentType = contentType;
+ if (build) runActOpts.build = build;
+ if (body) runActOpts.build = body;
+
+ // GetAct() options.
+ const { timeoutSecs = DEFAULT_APIFY_CALL_WAIT_SECS, fetchOutput = true } = opts;
+ checkParamOrThrow(timeoutSecs, 'timeoutSecs', 'Number');
+ checkParamOrThrow(fetchOutput, 'fetchOutput', 'Boolean');
+ const timeoutMillis = timeoutSecs * 1000;
+ const timeoutAt = Date.now() + timeoutMillis;
+
+ // GetRecord() options.
+ const { raw, useRawBody } = opts;
+ checkParamOrThrow(raw, 'raw', 'Maybe Boolean');
+ checkParamOrThrow(useRawBody, 'useRawBody', 'Maybe Boolean');
+
+ // Adds run.output field to given run and returns it.
+ const addOutputToRun = (run) => {
+ const getRecordOpts = { key: 'OUTPUT', storeId: run.defaultKeyValueStoreId };
+ if (raw) getRecordOpts.raw = raw;
+ if (useRawBody) getRecordOpts.useRawBody = useRawBody;
+
+ return keyValueStores
+ .getRecord(getRecordOpts)
+ .then(output => Object.assign({}, run, { output }));
+ };
+
+ // Keeps requesting given run until it gets finished or timeout is reached.
+ const waitForRunToFinish = (run) => {
+ const waitForFinish = Math.min(MAX_WAIT_FOR_FINISH_SECS, Math.round((timeoutAt - Date.now()) / 1000));
+
+ // We are timing out ...
+ if (waitForFinish <= 0) return Promise.resolve(run);
+
+ return acts
+ .getRun(Object.assign({}, defaultOpts, { waitForFinish, runId: run.id }))
+ .then((updatedRun) => {
+ if (!_.contains(ACT_TASK_TERMINAL_STATUSES, updatedRun.status)) return waitForRunToFinish(updatedRun);
+ if (!fetchOutput) return run;
+
+ return addOutputToRun(updatedRun);
+ });
+ };
+
+ return acts
+ .runAct(Object.assign({}, defaultOpts, runActOpts))
+ .then(run => waitForRunToFinish(run));
+};
diff --git a/src/constants.js b/src/constants.js
index 470f44547..70a63a17b 100644
--- a/src/constants.js
+++ b/src/constants.js
@@ -1,4 +1,9 @@
+/**
+ * Default wait of Apify.call() method.
+ */
+export const DEFAULT_APIFY_CALL_WAIT_SECS = 180;
+
/**
* Exit codes for the act process.
* The error codes must be in range 1-128, to avoid collision with signal exits
@@ -27,8 +32,31 @@ export const ENV_VARS = {
INTERNAL_PORT: 'APIFY_INTERNAL_PORT',
};
-
export const KEY_VALUE_STORE_KEYS = {
INPUT: 'INPUT',
OUTPUT: 'OUTPUT',
};
+
+/**
+ * Dictionary of possible values for 'status' field of act2Builds or act2Runs collections.
+ */
+export const ACT_TASK_STATUSES = {
+ READY: 'READY', // started but not allocated to any worker yet
+ RUNNING: 'RUNNING', // running on worker
+ SUCCEEDED: 'SUCCEEDED', // finished and all good
+ FAILED: 'FAILED', // run or build failed
+ TIMING_OUT: 'TIMING-OUT', // timing out now
+ TIMED_OUT: 'TIMED-OUT', // timed out
+ ABORTING: 'ABORTING', // being aborted by user
+ ABORTED: 'ABORTED', // aborted by user
+};
+
+/**
+ * An array of act task statuses that are final for the task.
+ */
+export const ACT_TASK_TERMINAL_STATUSES = [
+ ACT_TASK_STATUSES.SUCCEEDED,
+ ACT_TASK_STATUSES.FAILED,
+ ACT_TASK_STATUSES.TIMED_OUT,
+ ACT_TASK_STATUSES.ABORTED,
+];
diff --git a/src/index.js b/src/index.js
index 7857de019..6724bbf18 100644
--- a/src/index.js
+++ b/src/index.js
@@ -1,7 +1,7 @@
import EventEmitter from 'events';
import { setPromisesDependency, getPromisesDependency } from './utils';
-import { main, readyFreddy, getEnv, getValue, setValue, apifyClient } from './actor';
+import { main, readyFreddy, getEnv, getValue, setValue, apifyClient, call } from './actor';
import { browse } from './browser';
/* globals module */
@@ -12,6 +12,7 @@ const Apify = {
getEnv,
getValue,
setValue,
+ call,
readyFreddy,
setPromisesDependency,
getPromisesDependency,
diff --git a/test/actor.js b/test/actor.js
index 805199e1c..858322e8b 100644
--- a/test/actor.js
+++ b/test/actor.js
@@ -4,6 +4,8 @@ import { expect } from 'chai';
import sinon from 'sinon';
import tmp from 'tmp';
import Promise from 'bluebird';
+import { ACT_TASK_STATUSES } from '../build/constants';
+import { MAX_WAIT_FOR_FINISH_SECS } from 'apify-client/build/acts';
// NOTE: test use of require() here because this is how its done in acts
const Apify = require('../build/index');
@@ -726,3 +728,76 @@ describe('Apify.readyFreddy()', () => {
return testWatchFileWillBecomeEmpty(process.env.APIFY_WATCH_FILE, 1000);
});
});
+
+describe('Apify.call()', () => {
+ it('works as expected', () => {
+ const actId = 'some-act-id';
+ const token = 'some-token';
+ const defaultKeyValueStoreId = 'some-store-id';
+ const run = { id: 'some-run-id', actId, defaultKeyValueStoreId };
+ const runningRun = Object.assign({}, run, { status: 'RUNNING' });
+ const finishedRun = Object.assign({}, run, { status: ACT_TASK_STATUSES.SUCCEEDED });
+ const output = 'some-output';
+ const expected = Object.assign({}, finishedRun, { output });
+
+ const actsMock = sinon.mock(Apify.client.acts);
+ actsMock.expects('runAct')
+ .withExactArgs({ token, actId })
+ .once()
+ .returns(Promise.resolve(runningRun));
+ actsMock.expects('getRun')
+ .withExactArgs({ token, actId, runId: run.id, waitForFinish: MAX_WAIT_FOR_FINISH_SECS })
+ .once()
+ .returns(Promise.resolve(runningRun));
+ actsMock.expects('getRun')
+ .withExactArgs({ token, actId, runId: run.id, waitForFinish: MAX_WAIT_FOR_FINISH_SECS })
+ .once()
+ .returns(Promise.resolve(finishedRun));
+
+ const keyValueStoresMock = sinon.mock(Apify.client.keyValueStores);
+ keyValueStoresMock.expects('getRecord')
+ .withExactArgs({ storeId: run.defaultKeyValueStoreId, key: 'OUTPUT' })
+ .once()
+ .returns(Promise.resolve(output));
+
+ return Apify
+ .call(actId, { token })
+ .then((callOutput) => {
+ expect(callOutput).to.be.eql(expected);
+ keyValueStoresMock.restore();
+ actsMock.restore();
+ });
+ });
+
+ it('timeouts as expected with unfinished run', () => {
+ const actId = 'some-act-id';
+ const token = 'some-token';
+ const defaultKeyValueStoreId = 'some-store-id';
+ const run = { id: 'some-run-id', actId, defaultKeyValueStoreId };
+ const runningRun = Object.assign({}, run, { status: 'RUNNING' });
+ const timeoutSecs = 1;
+
+ const actsMock = sinon.mock(Apify.client.acts);
+ actsMock.expects('runAct')
+ .withExactArgs({ token, actId })
+ .once()
+ .returns(Promise.resolve(runningRun));
+ actsMock.expects('getRun')
+ .withExactArgs({ token, actId, runId: run.id, waitForFinish: timeoutSecs })
+ .once()
+ .returns(new Promise((resolve) => {
+ setTimeout(() => resolve(runningRun), timeoutSecs * 1000);
+ }));
+
+ const keyValueStoresMock = sinon.mock(Apify.client.keyValueStores);
+ keyValueStoresMock.expects('getRecord').never();
+
+ return Apify
+ .call(actId, { token, timeoutSecs })
+ .then((callOutput) => {
+ expect(callOutput).to.be.eql(runningRun);
+ keyValueStoresMock.restore();
+ actsMock.restore();
+ });
+ });
+});