Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Adding input and option overrides to actor tasks
Marek Trunkatc80bf2f49ec5
diff --git a/src/actor.js b/src/actor.jsindex 391134a6a..5d0377b2b 100644--- a/src/actor.js+++ b/src/actor.js@@ -70,6 +70,24 @@ const waitForRunToFinish = async ({ actId, runId, token, waitSecs, taskId }) =>return updatedRun;};+/**+ * Parses input and contentType and appends it to a given options object.+ * Throws if input is not valid.+ *+ * @ignore+ */+const addInputOptionsOrThrow = (input, contentType, options) => {+ options.contentType = contentType;++ // NOTE: this function modifies contentType property on options object if needed.+ options.body = maybeStringify(input, options);++ checkParamOrThrow(options.body, 'input', 'Buffer|String');+ checkParamOrThrow(options.contentType, 'contentType', 'String');++ if (options.contentType) options.contentType = addCharsetToContentType(options.contentType);+};+/*** Returns a new object which contains information parsed from the `APIFY_XXX` environment variables.* It has the following properties:@@ -319,19 +337,17 @@ let callMemoryWarningIssued = false;*/export const call = async (actId, input, options = {}) => {const { acts, keyValueStores } = apifyClient;- // Use optionsCopy here as maybeStringify() may override contentType- const optionsCopy = Object.assign({}, options);checkParamOrThrow(actId, 'actId', 'String');checkParamOrThrow(options, 'opts', 'Object');// Common options.- const { token } = optionsCopy;+ const { token } = options;checkParamOrThrow(token, 'token', 'Maybe String');// RunAct() options.- const { build, memory, timeoutSecs } = optionsCopy;- let { memoryMbytes } = optionsCopy;+ const { build, memory, timeoutSecs } = options;+ let { memoryMbytes } = options;const runActOpts = {actId,};@@ -353,15 +369,7 @@ export const call = async (actId, input, options = {}) => {if (build) runActOpts.build = build;if (memoryMbytes) runActOpts.memory = memoryMbytes;if (timeoutSecs >= 0) runActOpts.timeout = timeoutSecs; // Zero is valid value!- if (input) {- input = maybeStringify(input, optionsCopy);-- checkParamOrThrow(input, 'input', 'Buffer|String');- checkParamOrThrow(optionsCopy.contentType, 'contentType', 'String');-- if (optionsCopy.contentType) runActOpts.contentType = addCharsetToContentType(optionsCopy.contentType);- runActOpts.body = input;- }+ if (input) addInputOptionsOrThrow(input, options.contentType, runActOpts);// Run actor.const { waitSecs } = options;@@ -424,16 +432,33 @@ export const call = async (actId, input, options = {}) => {* @param {String} taskId* Either `username/task-name` or task ID.* @param {Object|String|Buffer} [input]- * This parameter is not supported yet. You must pass either `null` or `undefined` value!+ * Input overrides for the actor task. If it is an object, it will be stringified to+ * JSON and its content type set to `application/json; charset=utf-8`.+ * Otherwise the `options.contentType` parameter must be provided.+ * Provided input will be merged with actor task input.* @param {Object} [options]* Object with the settings below:+ * @param {String} [options.contentType]+ * Content type for the `input`. If not specified,+ * `input` is expected to be an object that will be stringified to JSON and content type set to+ * `application/json; charset=utf-8`. If `options.contentType` is specified, then `input` must be a+ * `String` or `Buffer`.* @param {String} [options.token]* User API token that is used to run the actor. By default, it is taken from the `APIFY_TOKEN` environment variable.+ * @param {Number} [options.memoryMbytes]+ * Memory in megabytes which will be allocated for the new actor task run.+ * If not provided, the run uses memory of the default actor run configuration.+ * @param {Number} [options.timeoutSecs]+ * Timeout for the actor task run in seconds. Zero value means there is no timeout.+ * If not provided, the run uses timeout of the default actor run configuration.+ * @param {String} [options.build]+ * Tag or number of the actor build to run (e.g. `beta` or `1.2.345`).+ * If not provided, the run uses build tag or number from the default actor run configuration (typically `latest`).* @param {String} [options.waitSecs]- * Maximum time to wait for the actor run to finish, in seconds.+ * Maximum time to wait for the actor task run to finish, in seconds.* If the limit is reached, the returned promise is resolved to a run object that will have* status `READY` or `RUNNING` and it will not contain the actor run output.- * If `waitSecs` is null or undefined, the function waits for the actor to finish (default behavior).+ * If `waitSecs` is null or undefined, the function waits for the actor task to finish (default behavior).* @returns {Promise<ActorRun>}* @throws {ApifyCallError} If the run did not succeed, e.g. if it failed or timed out.*@@ -444,8 +469,6 @@ export const call = async (actId, input, options = {}) => {export const callTask = async (taskId, input, options = {}) => {const { tasks, keyValueStores } = apifyClient;- if (input) throw new Error('Parameter "input" of Apify.callTask() is not supported yet!');-checkParamOrThrow(taskId, 'taskId', 'String');checkParamOrThrow(options, 'opts', 'Object');@@ -453,12 +476,22 @@ export const callTask = async (taskId, input, options = {}) => {const { token } = options;checkParamOrThrow(token, 'token', 'Maybe String');- // Run task.+ // Run task options.+ const { build, memoryMbytes, timeoutSecs } = options;+ const runTaskOpts = { taskId };+ checkParamOrThrow(build, 'build', 'Maybe String');+ checkParamOrThrow(memoryMbytes, 'memoryMbytes', 'Maybe Number');+ checkParamOrThrow(timeoutSecs, 'timeoutSecs', 'Maybe Number');+ if (token) runTaskOpts.token = token;+ if (build) runTaskOpts.build = build;+ if (memoryMbytes) runTaskOpts.memory = memoryMbytes;+ if (timeoutSecs >= 0) runTaskOpts.timeout = timeoutSecs; // Zero is valid value!+ if (input) addInputOptionsOrThrow(input, options.contentType, runTaskOpts);++ // Start task.const { waitSecs } = options;checkParamOrThrow(waitSecs, 'waitSecs', 'Maybe Number');- const opts = { taskId };- if (token) opts.token = token;- const run = await tasks.runTask(opts);+ const run = await tasks.runTask(runTaskOpts);if (waitSecs <= 0) return run; // In this case there is nothing more to do.// Wait for run to finish.diff --git a/test/actor.js b/test/actor.jsindex b3e476706..2a64c76ab 100644--- a/test/actor.js+++ b/test/actor.js@@ -713,10 +713,22 @@ describe('Apify.callTask()', () => {const finishedRun = Object.assign({}, run, { status: ACT_JOB_STATUSES.SUCCEEDED });const output = { contentType: 'application/json', body: 'some-output' };const expected = Object.assign({}, finishedRun, { output });+ const input = { foo: 'bar' };+ const memoryMbytes = 256;+ const timeoutSecs = 60;+ const build = 'beta';const tasksMock = sinon.mock(Apify.client.tasks);tasksMock.expects('runTask')- .withExactArgs({ token, taskId })+ .withExactArgs({+ token,+ taskId,+ body: JSON.stringify(input, null, 2),+ contentType: 'application/json; charset=utf-8',+ memory: memoryMbytes,+ timeout: timeoutSecs,+ build,+ }).once().returns(Promise.resolve(runningRun));@@ -737,7 +749,7 @@ describe('Apify.callTask()', () => {.returns(Promise.resolve(output));return Apify- .callTask(taskId, undefined, { token, disableBodyParser: true })+ .callTask(taskId, input, { token, disableBodyParser: true, memoryMbytes, timeoutSecs, build }).then((callOutput) => {expect(callOutput).to.be.eql(expected);keyValueStoresMock.restore();