Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Feature/apify call throw (#66)
Marek Trunkát69166cb1dcd0
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex d20838c68..dc53d9130 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -1,3 +1,7 @@+xxxxxxxxxxxxxxxxxxx+===================+- `Apify.call()` now throws an `ApifyCallError` error if run doesn't succeed+0.5.22 / 2018-04-12===================- `handleFailedRequestFunction` in both `BasicCrawler` and `PuppeteerCrawler` has now also error objectdiff --git a/src/actor.js b/src/actor.jsindex f8b174720..19de1abdc 100644--- a/src/actor.js+++ b/src/actor.js@@ -2,10 +2,11 @@ import _ from 'underscore';import Promise from 'bluebird';import fs from 'fs';import { checkParamOrThrow } from 'apify-client/build/utils';-import { ENV_VARS, EXIT_CODES, ACT_TASK_TERMINAL_STATUSES, DEFAULT_PROXY_HOSTNAME, DEFAULT_PROXY_PORT } from './constants';+import { ENV_VARS, EXIT_CODES, ACT_TASK_TERMINAL_STATUSES, ACT_TASK_STATUSES, DEFAULT_PROXY_HOSTNAME, DEFAULT_PROXY_PORT } from './constants';import { initializeEvents, stopEvents } from './events';import { newPromise, apifyClient, addCharsetToContentType } from './utils';import { maybeStringify } from './key_value_store';+import { ApifyCallError } from './errors';/* globals process */@@ -277,6 +278,7 @@ export const readyFreddy = () => {* @param {Boolean} [opts.disableBodyParser=false] If `true` then the function will not attempt to parse the* act's output and will return it in a raw `Buffer`.* @returns {Promise}+ * @throws {ApifyCallError} If run doesn't succeed.** @memberof module:Apify* @function@@ -353,6 +355,7 @@ export const call = (actId, input, opts = {}) => {}if (!_.contains(ACT_TASK_TERMINAL_STATUSES, updatedRun.status)) return waitForRunToFinish(updatedRun);+ if (updatedRun.status !== ACT_TASK_STATUSES.SUCCEEDED) throw new ApifyCallError(updatedRun);if (!fetchOutput) return updatedRun;return addOutputToRun(updatedRun);diff --git a/src/errors.js b/src/errors.jsnew file mode 100644index 000000000..5614ed308--- /dev/null+++ b/src/errors.js@@ -0,0 +1,18 @@+export const APIFY_CALL_ERROR_NAME = 'APIFY_CALL_ERROR';+export const APIFY_CALL_ERROR_MESSAGE = 'Apify.call() wasn\'t succeed';++/**+ * @typedef {Object} ApifyCallError+ * @property {String} message=Apify.call() wasn't succeed+ * @property {String} name=APIFY_CALL_ERROR+ * @property {Object} run Object of the failed run.+ */+export class ApifyCallError extends Error {+ constructor(run) {+ super(APIFY_CALL_ERROR_MESSAGE);+ this.name = APIFY_CALL_ERROR_NAME;+ this.run = run;++ Error.captureStackTrace(this, ApifyCallError);+ }+}diff --git a/test/actor.js b/test/actor.jsindex 6df0b453c..fa3297f17 100644--- a/test/actor.js+++ b/test/actor.js@@ -8,6 +8,7 @@ import tmp from 'tmp';import Promise from 'bluebird';import { delayPromise } from 'apify-shared/utilities';import { ACT_TASK_STATUSES, ENV_VARS, DEFAULT_PROXY_PORT, DEFAULT_PROXY_HOSTNAME } from '../build/constants';+import { ApifyCallError } from '../build/errors';chai.use(chaiAsPromised);@@ -331,7 +332,7 @@ describe('Apify.call()', () => {const token = 'some-token';const defaultKeyValueStoreId = 'some-store-id';const run = { id: 'some-run-id', actId, defaultKeyValueStoreId };- const runningRun = Object.assign({}, run, { status: 'RUNNING' });+ const runningRun = Object.assign({}, run, { status: ACT_TASK_STATUSES.RUNNING });const finishedRun = Object.assign({}, run, { status: ACT_TASK_STATUSES.SUCCEEDED });const input = 'something';const contentType = 'text/plain';@@ -373,7 +374,7 @@ describe('Apify.call()', () => {const token = 'token';const defaultKeyValueStoreId = 'some-store-id';const run = { id: 'some-run-id', actId, defaultKeyValueStoreId };- const runningRun = Object.assign({}, run, { status: 'RUNNING' });+ const runningRun = Object.assign({}, run, { status: ACT_TASK_STATUSES.RUNNING });const finishedRun = Object.assign({}, run, { status: ACT_TASK_STATUSES.SUCCEEDED });const output = 'some-output';const expected = Object.assign({}, finishedRun, { output });@@ -414,7 +415,7 @@ describe('Apify.call()', () => {const token = 'token';const defaultKeyValueStoreId = 'some-store-id';const run = { id: 'some-run-id', actId, defaultKeyValueStoreId };- const runningRun = Object.assign({}, run, { status: 'RUNNING' });+ const runningRun = Object.assign({}, run, { status: ACT_TASK_STATUSES.RUNNING });const finishedRun = Object.assign({}, run, { status: ACT_TASK_STATUSES.SUCCEEDED });const output = 'some-output';const expected = Object.assign({}, finishedRun, { output });@@ -455,7 +456,7 @@ describe('Apify.call()', () => {const token = 'token';const defaultKeyValueStoreId = 'some-store-id';const run = { id: 'some-run-id', actId, defaultKeyValueStoreId };- const runningRun = Object.assign({}, run, { status: 'RUNNING' });+ const runningRun = Object.assign({}, run, { status: ACT_TASK_STATUSES.RUNNING });const finishedRun = Object.assign({}, run, { status: ACT_TASK_STATUSES.SUCCEEDED });const input = { a: 'b' };const output = 'some-output';@@ -498,7 +499,7 @@ describe('Apify.call()', () => {const token = 'some-token';const defaultKeyValueStoreId = 'some-store-id';const run = { id: 'some-run-id', actId, defaultKeyValueStoreId };- const runningRun = Object.assign({}, run, { status: 'RUNNING' });+ const runningRun = Object.assign({}, run, { status: ACT_TASK_STATUSES.RUNNING });const finishedRun = Object.assign({}, run, { status: ACT_TASK_STATUSES.SUCCEEDED });const input = { a: 'b' };const output = { body: 'some-output' };@@ -539,7 +540,7 @@ describe('Apify.call()', () => {const token = 'some-token';const defaultKeyValueStoreId = 'some-store-id';const run = { id: 'some-run-id', actId, defaultKeyValueStoreId };- const runningRun = Object.assign({}, run, { status: 'RUNNING' });+ const runningRun = Object.assign({}, run, { status: ACT_TASK_STATUSES.RUNNING });const finishedRun = Object.assign({}, run, { status: ACT_TASK_STATUSES.SUCCEEDED });const actsMock = sinon.mock(Apify.client.acts);@@ -573,7 +574,7 @@ describe('Apify.call()', () => {const token = 'some-token';const defaultKeyValueStoreId = 'some-store-id';const run = { id: 'some-run-id', actId, defaultKeyValueStoreId };- const runningRun = Object.assign({}, run, { status: 'RUNNING' });+ const runningRun = Object.assign({}, run, { status: ACT_TASK_STATUSES.RUNNING });const waitSecs = 1;const actsMock = sinon.mock(Apify.client.acts);@@ -605,7 +606,7 @@ describe('Apify.call()', () => {const token = 'some-token';const defaultKeyValueStoreId = 'some-store-id';const run = { id: 'some-run-id', actId, defaultKeyValueStoreId };- const runningRun = Object.assign({}, run, { status: 'RUNNING' });+ const runningRun = Object.assign({}, run, { status: ACT_TASK_STATUSES.RUNNING });const finishedRun = Object.assign({}, run, { status: ACT_TASK_STATUSES.SUCCEEDED });const input = 'something';const contentType = 'text/plain';@@ -647,7 +648,7 @@ describe('Apify.call()', () => {const token = 'some-token';const defaultKeyValueStoreId = 'some-store-id';const run = { id: 'some-run-id', actId, defaultKeyValueStoreId };- const readyRun = Object.assign({}, run, { status: 'READY' });+ const readyRun = Object.assign({}, run, { status: ACT_TASK_STATUSES.READY });const waitSecs = 0;const actsMock = sinon.mock(Apify.client.acts);@@ -668,6 +669,33 @@ describe('Apify.call()', () => {actsMock.restore();});});++ it('throws if run doesn\'t succeed', () => {+ const actId = 'some-act-id';+ const token = 'some-token';+ const run = { id: 'some-run-id' };+ const runningRun = Object.assign({}, run, { status: ACT_TASK_STATUSES.RUNNING });+ const failedRun = Object.assign({}, run, { status: ACT_TASK_STATUSES.ABORTED });++ const actsMock = sinon.mock(Apify.client.acts);+ actsMock.expects('runAct')+ .withExactArgs({ token, actId })+ .once()+ .returns(Promise.resolve(runningRun));+ actsMock.expects('getRun')+ .withExactArgs({ token, actId, runId: run.id, waitForFinish: 999999 })+ .once()+ .returns(Promise.resolve(failedRun));++ return Apify+ .call(actId, null, { token })+ .then(() => { throw new Error('This was suppose to fail!'); }, (err) => {+ expect(err).to.be.instanceOf(ApifyCallError);+ console.log(err);+ expect(err.run.status).to.be.eql(ACT_TASK_STATUSES.ABORTED);+ expect(err.run).to.be.eql(failedRun);+ });+ });});describe('Apify.getApifyProxyUrl()', () => {