Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Last few changes of callTask() and call()
Marek Trunkat45ffc3cd8ee6
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 15aa4a220..3d29b2c24 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -3,6 +3,9 @@ xxx- Fixed `RequestQueue.addRequest()` to gracefully handle invalid URLs- Renamed `RequestOperationInfo` to `QueueOperationInfo`- Added `request` field to `QueueOperationInfo`+- **WARNING**: Deprecated parameter `timeoutSecs` of `Apify.call()` was removed use `waitSecs` instead.+ Parameter `timeout` is used for actor run timeout.+- Added `Apify.callTask()` that enables to start actor task and fetch its output.0.8.18 / 2018-10-30===================diff --git a/src/actor.js b/src/actor.jsindex 0ff148786..f83a1369a 100644--- a/src/actor.js+++ b/src/actor.js@@ -25,25 +25,25 @@ const tryParseDate = (str) => {};/**- * Waits for given run to finish. If "timeoutSecs" is reached then returns unfinished run.+ * Waits for given run to finish. If "waitSecs" is reached then returns unfinished run.** @ignore*/-const waitForRunToFinish = async ({ actId, runId, token, timeoutSecs }) => {+const waitForRunToFinish = async ({ actId, runId, token, waitSecs }) => {let updatedRun;const { acts } = apifyClient;const startedAt = Date.now();const shouldRepeat = () => {- if (timeoutSecs && (Date.now() - startedAt) / 1000 >= timeoutSecs) return false;+ if (waitSecs && (Date.now() - startedAt) / 1000 >= waitSecs) return false;if (updatedRun && ACT_JOB_TERMINAL_STATUSES.includes(updatedRun.status)) return false;return true;};while (shouldRepeat()) {- const waitForFinish = timeoutSecs- ? Math.round(timeoutSecs - (Date.now() - startedAt) / 1000)+ const waitForFinish = waitSecs+ ? Math.round(waitSecs - (Date.now() - startedAt) / 1000): 999999;updatedRun = await acts.getRun({ actId, runId, token, waitForFinish });@@ -276,14 +276,17 @@ export const main = (userFunc) => {* @param {Number} [options.memory]* Memory in megabytes which will be allocated for the new actor run.* If not provided, the run uses memory of the default actor run configuration.+ * @param {Number} [options.timeout]+ * Timeout for the act run in seconds. Zero value means there is no timeout.+ * If not provided, the run uses timeout of the default actor run configuration.* @param {String} [options.build]* Tag or number of the actor build to run (e.g. `beta` or `1.2.345`).* If not provided, the run uses build tag or number from the default actor run configuration (typically `latest`).- * @param {String} [options.timeoutSecs]+ * @param {String} [options.waitSecs]* Maximum time to wait for the actor run to finish, in seconds.* If the limit is reached, the returned promise is resolved to a run object that will have* status `READY` or `RUNNING` and it will not contain the actor run output.- * If `timeoutSecs` is null or undefined, the function waits for the actor to finish (default behavior).+ * If `waitSecs` is null or undefined, the function waits for the actor to finish (default behavior).* @param {Boolean} [options.fetchOutput=true]* If `false` then the function does not fetch output of the actor.* @param {Boolean} [options.disableBodyParser=false]@@ -309,12 +312,14 @@ export const call = async (actId, input, options = {}) => {checkParamOrThrow(token, 'token', 'Maybe String');// RunAct() options.- const { build, memory } = optionsCopy;+ const { build, memory, timeout } = optionsCopy;const runActOpts = {};checkParamOrThrow(build, 'build', 'Maybe String');checkParamOrThrow(memory, 'memory', 'Maybe Number');+ checkParamOrThrow(timeout, 'timeout', 'Maybe Number');if (build) runActOpts.build = build;if (memory) runActOpts.memory = memory;+ if (timeout >= 0) runActOpts.timeout = timeout; // Zero is valid value!if (input) {input = maybeStringify(input, optionsCopy);@@ -326,17 +331,17 @@ export const call = async (actId, input, options = {}) => {}// Run actor.- const { timeoutSecs } = options;- checkParamOrThrow(timeoutSecs, 'timeoutSecs', 'Maybe Number');+ const { waitSecs } = options;+ checkParamOrThrow(waitSecs, 'waitSecs', 'Maybe Number');const run = await acts.runAct(Object.assign({ actId, token }, runActOpts));- if (timeoutSecs <= 0) return run; // In this case there is nothing more to do.+ if (waitSecs <= 0) return run; // In this case there is nothing more to do.// Wait for run to finish.const updatedRun = await waitForRunToFinish({actId,runId: run.id,token,- timeoutSecs,+ waitSecs,});// Finish if output is not requested or run haven't finished.@@ -385,11 +390,11 @@ export const call = async (actId, input, options = {}) => {* Object with the settings below:* @param {String} [options.token]* User API token that is used to run the actor. By default, it is taken from the `APIFY_TOKEN` environment variable.- * @param {String} [options.timeoutSecs]+ * @param {String} [options.waitSecs]* Maximum time to wait for the actor run to finish, in seconds.* If the limit is reached, the returned promise is resolved to a run object that will have* status `READY` or `RUNNING` and it will not contain the actor run output.- * If `timeoutSecs` is null or undefined, the function waits for the actor to finish (default behavior).+ * If `waitSecs` is null or undefined, the function waits for the actor to finish (default behavior).* @returns {Promise<ActorRun>}* @throws {ApifyCallError} If the run did not succeed, e.g. if it failed or timed out.*@@ -410,17 +415,17 @@ export const callTask = async (taskId, input, options = {}) => {checkParamOrThrow(token, 'token', 'Maybe String');// Run task.- const { timeoutSecs } = options;- checkParamOrThrow(timeoutSecs, 'timeoutSecs', 'Maybe Number');+ const { waitSecs } = options;+ checkParamOrThrow(waitSecs, 'waitSecs', 'Maybe Number');const run = await tasks.runTask({ taskId, token });- if (timeoutSecs <= 0) return run; // In this case there is nothing more to do.+ if (waitSecs <= 0) return run; // In this case there is nothing more to do.// Wait for run to finish.const updatedRun = await waitForRunToFinish({actId: run.actId,runId: run.id,token,- timeoutSecs,+ waitSecs,});// Finish if output is not requested or run haven't finished.@@ -486,7 +491,7 @@ export const callTask = async (taskId, input, options = {}) => {* ```* {* "build": "latest",- * "timeoutSecs": 0,+ * "waitSecs": 0,* "memoryMbytes": 256,* "diskMbytes": 512* }diff --git a/test/actor.js b/test/actor.jsindex d00e12831..5f2c43055 100644--- a/test/actor.js+++ b/test/actor.js@@ -244,10 +244,11 @@ describe('Apify.call()', () => {const expected = Object.assign({}, finishedRun, { output });const build = 'xxx';const memory = 1024;+ const timeout = 60;const actsMock = sinon.mock(Apify.client.acts);actsMock.expects('runAct')- .withExactArgs({ token, actId, contentType: `${contentType}; charset=utf-8`, body: input, build, memory })+ .withExactArgs({ token, actId, contentType: `${contentType}; charset=utf-8`, body: input, build, memory, timeout }).once().returns(Promise.resolve(runningRun));actsMock.expects('getRun')@@ -266,7 +267,7 @@ describe('Apify.call()', () => {.returns(Promise.resolve(output));return Apify- .call(actId, input, { contentType, token, disableBodyParser: true, build, memory })+ .call(actId, input, { contentType, token, disableBodyParser: true, build, memory, timeout }).then((callOutput) => {expect(callOutput).to.be.eql(expected);keyValueStoresMock.restore();@@ -480,7 +481,7 @@ describe('Apify.call()', () => {const defaultKeyValueStoreId = 'some-store-id';const run = { id: 'some-run-id', actId, defaultKeyValueStoreId };const runningRun = Object.assign({}, run, { status: ACT_JOB_STATUSES.RUNNING });- const timeoutSecs = 1;+ const waitSecs = 1;const actsMock = sinon.mock(Apify.client.acts);actsMock.expects('runAct')@@ -488,17 +489,17 @@ describe('Apify.call()', () => {.once().returns(Promise.resolve(runningRun));actsMock.expects('getRun')- .withExactArgs({ token, actId, runId: run.id, waitForFinish: timeoutSecs })+ .withExactArgs({ token, actId, runId: run.id, waitForFinish: waitSecs }).once().returns(new Promise((resolve) => {- setTimeout(() => resolve(runningRun), timeoutSecs * 1000);+ setTimeout(() => resolve(runningRun), waitSecs * 1000 * 2);}));const keyValueStoresMock = sinon.mock(Apify.client.keyValueStores);keyValueStoresMock.expects('getRecord').never();return Apify- .call(actId, null, { token, timeoutSecs })+ .call(actId, null, { token, waitSecs }).then((callOutput) => {expect(callOutput).to.be.eql(runningRun);keyValueStoresMock.restore();@@ -554,7 +555,7 @@ describe('Apify.call()', () => {const defaultKeyValueStoreId = 'some-store-id';const run = { id: 'some-run-id', actId, defaultKeyValueStoreId };const readyRun = Object.assign({}, run, { status: ACT_JOB_STATUSES.READY });- const timeoutSecs = 0;+ const waitSecs = 0;const actsMock = sinon.mock(Apify.client.acts);actsMock.expects('runAct')@@ -567,7 +568,7 @@ describe('Apify.call()', () => {keyValueStoresMock.expects('getRecord').never();return Apify- .call(actId, null, { token, timeoutSecs })+ .call(actId, null, { token, waitSecs }).then((callOutput) => {expect(callOutput).to.be.eql(readyRun);keyValueStoresMock.restore();@@ -686,7 +687,7 @@ describe('Apify.callTask()', () => {});it('timeouts as expected with unfinished run', () => {- const timeoutSecs = 1;+ const waitSecs = 1;const taskId = 'some-act-id';const actId = 'xxx';const token = 'some-token';@@ -702,17 +703,17 @@ describe('Apify.callTask()', () => {const actsMock = sinon.mock(Apify.client.acts);actsMock.expects('getRun')- .withExactArgs({ token, actId, runId: run.id, waitForFinish: timeoutSecs })+ .withExactArgs({ token, actId, runId: run.id, waitForFinish: waitSecs }).once().returns(new Promise((resolve) => {- setTimeout(() => resolve(runningRun), timeoutSecs * 1000);+ setTimeout(() => resolve(runningRun), waitSecs * 1000 * 2);}));const keyValueStoresMock = sinon.mock(Apify.client.keyValueStores);keyValueStoresMock.expects('getRecord').never();return Apify- .callTask(taskId, undefined, { token, disableBodyParser: true, fetchOutput: false, timeoutSecs })+ .callTask(taskId, undefined, { token, disableBodyParser: true, fetchOutput: false, waitSecs }).then((callOutput) => {expect(callOutput).to.be.eql(runningRun);keyValueStoresMock.restore();@@ -766,7 +767,7 @@ describe('Apify.callTask()', () => {});it('returns immediately with zero timeout', () => {- const timeoutSecs = 0;+ const waitSecs = 0;const taskId = 'some-act-id';const actId = 'xxx';const token = 'some-token';@@ -787,7 +788,7 @@ describe('Apify.callTask()', () => {keyValueStoresMock.expects('getRecord').never();return Apify- .callTask(taskId, undefined, { token, timeoutSecs })+ .callTask(taskId, undefined, { token, waitSecs }).then((callOutput) => {expect(callOutput).to.be.eql(readyRun);keyValueStoresMock.restore();