Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Feature/call wait (#28)
Jan Čurn15407c13ca8f
diff --git a/src/actor.js b/src/actor.jsindex 744e82287..290fb7232 100644--- a/src/actor.js+++ b/src/actor.js@@ -202,9 +202,13 @@ export const readyFreddy = () => {};/**- * Executes another act under the current user account, waits for the act finish and fetches its output.+ * Runs another act under the current user account, waits for the act to finish and fetches its output.*- * The result of the function is an object describing the act run, which looks something like this:+ * By passing the `waitSecs` option you can reduce the maximum amount of time to wait for the run to finish.+ * If the value is less than or equal to zero, the function returns immediately after the run is started.+ *+ * The result of the function is an object that contains details about the run and potentially its output.+ * For example:** ```json* {@@ -229,6 +233,7 @@ export const readyFreddy = () => {* "buildId": "Bwkqk59MCkdexDP34",* "exitCode": 0,* "defaultKeyValueStoreId": "ccFfRptZru2uqdQHP",+ * "defaultDatasetId": "tZru2uqdQHPcgFtRo",* "buildNumber": "0.1.2",* "output": {* "contentType": "application/json; charset=utf-8",@@ -260,6 +265,10 @@ export const readyFreddy = () => {* @param {Number} [opts.timeoutSecs] Time limit for act to finish, in seconds.* If the limit is reached the resulting run will have the `RUNNING` status.* By default, there is no timeout.+ * @param {String} [opts.waitSecs] - Maximum time to wait for act run to finish, in seconds.+ * If the limit is reached, the returned promise is resolved to a run object that will have+ * status `READY` or `RUNNING` and it will not contain the act run output.+ * If `waitSecs` is null or undefined, the function waits for the act to finish (default behavior).* @param {Boolean} [opts.fetchOutput=true] If `false` then the function does not fetch output of the act.* @param {Boolean} [opts.disableBodyParser=false] If `true` then the function will not attempt to parse the* act's output and will return it in a raw `Buffer`.@@ -312,9 +321,12 @@ export const call = (actId, input, opts = {}) => {// GetAct() options.const { timeoutSecs, fetchOutput = true } = opts;- checkParamOrThrow(timeoutSecs, 'timeoutSecs', 'Maybe Number');+ let { waitSecs } = opts;+ // Backwards compatibility: waitSecs used to be called timeoutSecs+ if (typeof timeoutSecs === 'number' && typeof waitSecs !== 'number') waitSecs = timeoutSecs;+ checkParamOrThrow(waitSecs, 'waitSecs', 'Maybe Number');checkParamOrThrow(fetchOutput, 'fetchOutput', 'Boolean');- const timeoutAt = timeoutSecs ? Date.now() + (timeoutSecs * 1000) : null;+ const waitUntil = typeof waitSecs === 'number' ? Date.now() + (waitSecs * 1000) : null;// GetRecord() options.const { disableBodyParser } = opts;@@ -332,7 +344,7 @@ export const call = (actId, input, opts = {}) => {// Keeps requesting given run until it gets finished or timeout is reached.const waitForRunToFinish = (run) => {- const waitForFinish = timeoutAt !== null ? Math.round((timeoutAt - Date.now()) / 1000) : 999999;+ const waitForFinish = waitUntil !== null ? Math.round((waitUntil - Date.now()) / 1000) : 999999;// We are timing out ...if (waitForFinish <= 0) return Promise.resolve(run);@@ -341,7 +353,7 @@ export const call = (actId, input, opts = {}) => {.getRun(Object.assign({}, defaultOpts, { waitForFinish, runId: run.id })).then((updatedRun) => {// It might take some time for database replicas to get up-to-date,- // so getRun() might return null. Wait a little while and try it again.+ // so getRun() might return null. Wait a little bit and try it again.if (!updatedRun) {return new Promise(resolve => setTimeout(resolve, 250)).then(() => {diff --git a/test/actor.js b/test/actor.jsindex d47a30a96..81fc0c602 100644--- a/test/actor.js+++ b/test/actor.js@@ -587,7 +587,7 @@ describe('Apify.call()', () => {const defaultKeyValueStoreId = 'some-store-id';const run = { id: 'some-run-id', actId, defaultKeyValueStoreId };const runningRun = Object.assign({}, run, { status: 'RUNNING' });- const timeoutSecs = 1;+ const waitSecs = 1;const actsMock = sinon.mock(Apify.client.acts);actsMock.expects('runAct')@@ -595,17 +595,17 @@ describe('Apify.call()', () => {.once().returns(Promise.resolve(runningRun));actsMock.expects('getRun')- .withExactArgs({ token, actId, runId: run.id, waitForFinish: timeoutSecs })+ .withExactArgs({ token, actId, runId: run.id, waitForFinish: waitSecs }).once().returns(new Promise((resolve) => {- setTimeout(() => resolve(runningRun), timeoutSecs * 1000);+ setTimeout(() => resolve(runningRun), waitSecs * 1000);}));const keyValueStoresMock = sinon.mock(Apify.client.keyValueStores);keyValueStoresMock.expects('getRecord').never();return Apify- .call(actId, null, { token, timeoutSecs })+ .call(actId, null, { token, waitSecs }).then((callOutput) => {expect(callOutput).to.be.eql(runningRun);keyValueStoresMock.restore();@@ -654,6 +654,33 @@ describe('Apify.call()', () => {actsMock.restore();});});++ it('returns immediately with zero timeout', () => {+ const actId = 'some-act-id';+ const token = 'some-token';+ const defaultKeyValueStoreId = 'some-store-id';+ const run = { id: 'some-run-id', actId, defaultKeyValueStoreId };+ const readyRun = Object.assign({}, run, { status: 'READY' });+ const waitSecs = 0;++ const actsMock = sinon.mock(Apify.client.acts);+ actsMock.expects('runAct')+ .withExactArgs({ token, actId })+ .once()+ .returns(Promise.resolve(readyRun));+ actsMock.expects('getRun').never();++ const keyValueStoresMock = sinon.mock(Apify.client.keyValueStores);+ keyValueStoresMock.expects('getRecord').never();++ return Apify+ .call(actId, null, { token, waitSecs })+ .then((callOutput) => {+ expect(callOutput).to.be.eql(readyRun);+ keyValueStoresMock.restore();+ actsMock.restore();+ });+ });});describe('Apify.getApifyProxyUrl()', () => {