Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Feature/apify call throw (#66)

Marek Trunkát69166cb1dcd0
diff --git a/CHANGELOG.md b/CHANGELOG.md
index d20838c68..dc53d9130 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,3 +1,7 @@
+xxxxxxxxxxxxxxxxxxx
+===================
+- `Apify.call()` now throws an `ApifyCallError` error if run doesn't succeed
+
0.5.22 / 2018-04-12
===================
- `handleFailedRequestFunction` in both `BasicCrawler` and `PuppeteerCrawler` has now also error object
diff --git a/src/actor.js b/src/actor.js
index f8b174720..19de1abdc 100644
--- a/src/actor.js
+++ b/src/actor.js
@@ -2,10 +2,11 @@ import _ from 'underscore';
import Promise from 'bluebird';
import fs from 'fs';
import { checkParamOrThrow } from 'apify-client/build/utils';
-import { ENV_VARS, EXIT_CODES, ACT_TASK_TERMINAL_STATUSES, DEFAULT_PROXY_HOSTNAME, DEFAULT_PROXY_PORT } from './constants';
+import { ENV_VARS, EXIT_CODES, ACT_TASK_TERMINAL_STATUSES, ACT_TASK_STATUSES, DEFAULT_PROXY_HOSTNAME, DEFAULT_PROXY_PORT } from './constants';
import { initializeEvents, stopEvents } from './events';
import { newPromise, apifyClient, addCharsetToContentType } from './utils';
import { maybeStringify } from './key_value_store';
+import { ApifyCallError } from './errors';
/* globals process */
@@ -277,6 +278,7 @@ export const readyFreddy = () => {
* @param {Boolean} [opts.disableBodyParser=false] If `true` then the function will not attempt to parse the
* act's output and will return it in a raw `Buffer`.
* @returns {Promise}
+ * @throws {ApifyCallError} If run doesn't succeed.
*
* @memberof module:Apify
* @function
@@ -353,6 +355,7 @@ export const call = (actId, input, opts = {}) => {
}
if (!_.contains(ACT_TASK_TERMINAL_STATUSES, updatedRun.status)) return waitForRunToFinish(updatedRun);
+ if (updatedRun.status !== ACT_TASK_STATUSES.SUCCEEDED) throw new ApifyCallError(updatedRun);
if (!fetchOutput) return updatedRun;
return addOutputToRun(updatedRun);
diff --git a/src/errors.js b/src/errors.js
new file mode 100644
index 000000000..5614ed308
--- /dev/null
+++ b/src/errors.js
@@ -0,0 +1,18 @@
+export const APIFY_CALL_ERROR_NAME = 'APIFY_CALL_ERROR';
+export const APIFY_CALL_ERROR_MESSAGE = 'Apify.call() wasn\'t succeed';
+
+/**
+ * @typedef {Object} ApifyCallError
+ * @property {String} message=Apify.call() wasn't succeed
+ * @property {String} name=APIFY_CALL_ERROR
+ * @property {Object} run Object of the failed run.
+ */
+export class ApifyCallError extends Error {
+ constructor(run) {
+ super(APIFY_CALL_ERROR_MESSAGE);
+ this.name = APIFY_CALL_ERROR_NAME;
+ this.run = run;
+
+ Error.captureStackTrace(this, ApifyCallError);
+ }
+}
diff --git a/test/actor.js b/test/actor.js
index 6df0b453c..fa3297f17 100644
--- a/test/actor.js
+++ b/test/actor.js
@@ -8,6 +8,7 @@ import tmp from 'tmp';
import Promise from 'bluebird';
import { delayPromise } from 'apify-shared/utilities';
import { ACT_TASK_STATUSES, ENV_VARS, DEFAULT_PROXY_PORT, DEFAULT_PROXY_HOSTNAME } from '../build/constants';
+import { ApifyCallError } from '../build/errors';
chai.use(chaiAsPromised);
@@ -331,7 +332,7 @@ describe('Apify.call()', () => {
const token = 'some-token';
const defaultKeyValueStoreId = 'some-store-id';
const run = { id: 'some-run-id', actId, defaultKeyValueStoreId };
- const runningRun = Object.assign({}, run, { status: 'RUNNING' });
+ const runningRun = Object.assign({}, run, { status: ACT_TASK_STATUSES.RUNNING });
const finishedRun = Object.assign({}, run, { status: ACT_TASK_STATUSES.SUCCEEDED });
const input = 'something';
const contentType = 'text/plain';
@@ -373,7 +374,7 @@ describe('Apify.call()', () => {
const token = 'token';
const defaultKeyValueStoreId = 'some-store-id';
const run = { id: 'some-run-id', actId, defaultKeyValueStoreId };
- const runningRun = Object.assign({}, run, { status: 'RUNNING' });
+ const runningRun = Object.assign({}, run, { status: ACT_TASK_STATUSES.RUNNING });
const finishedRun = Object.assign({}, run, { status: ACT_TASK_STATUSES.SUCCEEDED });
const output = 'some-output';
const expected = Object.assign({}, finishedRun, { output });
@@ -414,7 +415,7 @@ describe('Apify.call()', () => {
const token = 'token';
const defaultKeyValueStoreId = 'some-store-id';
const run = { id: 'some-run-id', actId, defaultKeyValueStoreId };
- const runningRun = Object.assign({}, run, { status: 'RUNNING' });
+ const runningRun = Object.assign({}, run, { status: ACT_TASK_STATUSES.RUNNING });
const finishedRun = Object.assign({}, run, { status: ACT_TASK_STATUSES.SUCCEEDED });
const output = 'some-output';
const expected = Object.assign({}, finishedRun, { output });
@@ -455,7 +456,7 @@ describe('Apify.call()', () => {
const token = 'token';
const defaultKeyValueStoreId = 'some-store-id';
const run = { id: 'some-run-id', actId, defaultKeyValueStoreId };
- const runningRun = Object.assign({}, run, { status: 'RUNNING' });
+ const runningRun = Object.assign({}, run, { status: ACT_TASK_STATUSES.RUNNING });
const finishedRun = Object.assign({}, run, { status: ACT_TASK_STATUSES.SUCCEEDED });
const input = { a: 'b' };
const output = 'some-output';
@@ -498,7 +499,7 @@ describe('Apify.call()', () => {
const token = 'some-token';
const defaultKeyValueStoreId = 'some-store-id';
const run = { id: 'some-run-id', actId, defaultKeyValueStoreId };
- const runningRun = Object.assign({}, run, { status: 'RUNNING' });
+ const runningRun = Object.assign({}, run, { status: ACT_TASK_STATUSES.RUNNING });
const finishedRun = Object.assign({}, run, { status: ACT_TASK_STATUSES.SUCCEEDED });
const input = { a: 'b' };
const output = { body: 'some-output' };
@@ -539,7 +540,7 @@ describe('Apify.call()', () => {
const token = 'some-token';
const defaultKeyValueStoreId = 'some-store-id';
const run = { id: 'some-run-id', actId, defaultKeyValueStoreId };
- const runningRun = Object.assign({}, run, { status: 'RUNNING' });
+ const runningRun = Object.assign({}, run, { status: ACT_TASK_STATUSES.RUNNING });
const finishedRun = Object.assign({}, run, { status: ACT_TASK_STATUSES.SUCCEEDED });
const actsMock = sinon.mock(Apify.client.acts);
@@ -573,7 +574,7 @@ describe('Apify.call()', () => {
const token = 'some-token';
const defaultKeyValueStoreId = 'some-store-id';
const run = { id: 'some-run-id', actId, defaultKeyValueStoreId };
- const runningRun = Object.assign({}, run, { status: 'RUNNING' });
+ const runningRun = Object.assign({}, run, { status: ACT_TASK_STATUSES.RUNNING });
const waitSecs = 1;
const actsMock = sinon.mock(Apify.client.acts);
@@ -605,7 +606,7 @@ describe('Apify.call()', () => {
const token = 'some-token';
const defaultKeyValueStoreId = 'some-store-id';
const run = { id: 'some-run-id', actId, defaultKeyValueStoreId };
- const runningRun = Object.assign({}, run, { status: 'RUNNING' });
+ const runningRun = Object.assign({}, run, { status: ACT_TASK_STATUSES.RUNNING });
const finishedRun = Object.assign({}, run, { status: ACT_TASK_STATUSES.SUCCEEDED });
const input = 'something';
const contentType = 'text/plain';
@@ -647,7 +648,7 @@ describe('Apify.call()', () => {
const token = 'some-token';
const defaultKeyValueStoreId = 'some-store-id';
const run = { id: 'some-run-id', actId, defaultKeyValueStoreId };
- const readyRun = Object.assign({}, run, { status: 'READY' });
+ const readyRun = Object.assign({}, run, { status: ACT_TASK_STATUSES.READY });
const waitSecs = 0;
const actsMock = sinon.mock(Apify.client.acts);
@@ -668,6 +669,33 @@ describe('Apify.call()', () => {
actsMock.restore();
});
});
+
+ it('throws if run doesn\'t succeed', () => {
+ const actId = 'some-act-id';
+ const token = 'some-token';
+ const run = { id: 'some-run-id' };
+ const runningRun = Object.assign({}, run, { status: ACT_TASK_STATUSES.RUNNING });
+ const failedRun = Object.assign({}, run, { status: ACT_TASK_STATUSES.ABORTED });
+
+ const actsMock = sinon.mock(Apify.client.acts);
+ actsMock.expects('runAct')
+ .withExactArgs({ token, actId })
+ .once()
+ .returns(Promise.resolve(runningRun));
+ actsMock.expects('getRun')
+ .withExactArgs({ token, actId, runId: run.id, waitForFinish: 999999 })
+ .once()
+ .returns(Promise.resolve(failedRun));
+
+ return Apify
+ .call(actId, null, { token })
+ .then(() => { throw new Error('This was suppose to fail!'); }, (err) => {
+ expect(err).to.be.instanceOf(ApifyCallError);
+ console.log(err);
+ expect(err.run.status).to.be.eql(ACT_TASK_STATUSES.ABORTED);
+ expect(err.run).to.be.eql(failedRun);
+ });
+ });
});
describe('Apify.getApifyProxyUrl()', () => {