Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Development of better act run input/output

Jan Curn05e119bb726b
diff --git a/src/actor.js b/src/actor.js
index 8ce6b4c6b..d03fb3512 100644
--- a/src/actor.js
+++ b/src/actor.js
@@ -1,9 +1,13 @@
import fs from 'fs';
+import ApifyClient from 'apify-client';
import { APIFY_ENV_VARS, EXIT_CODES } from './constants';
-import { newPromise } from './utils';
+import { getPromisesDependency, newPromise } from './utils';
/* global process */
+const apifyClient = new ApifyClient();
+
+
/**
* Tries to parse a string with date.
* @param str Date string
@@ -14,33 +18,101 @@ const tryParseDate = (str) => {
return unix > 0 ? new Date(unix) : undefined;
};
+const getDefaultStoreIdOrThrow = () => {
+ const storeId = process.env[APIFY_ENV_VARS.DEFAULT_KEY_VALUE_STORE_ID];
+ if (!storeId) throw new Error(`The '${APIFY_ENV_VARS.DEFAULT_KEY_VALUE_STORE_ID}' environment variable is not defined.`);
+ return storeId;
+};
+
+
/**
- * Gets a context object which contains meta-data about this act run
- * extracted from the APIFY_XXX environment variables.
- * @return Returns a dictionary with all properties that could be determined from environment variables.
+ * Gets input data for the current act run.
+ * This data is stored in the key-value store created specifically for this run,
+ * whose ID is defined in the `APIFY_DEFAULT_KEY_VALUE_STORE_ID` environment variable,
+ * under record with the `INPUT` key.
+ * The result of the function is an object such as `{ body: String/Buffer, contentType: String }`,
+ * or `null` if record was not found.
+ * @param callback Optional callback.
+ * @return Returns a promise if no callback was provided, otherwise the return value is not defined.
*/
-export const getContext = () => {
- return {
- internalPort: process.env[APIFY_ENV_VARS.INTERNAL_PORT],
- actId: process.env[APIFY_ENV_VARS.ACT_ID],
- actRunId: process.env[APIFY_ENV_VARS.ACT_RUN_ID],
- startedAt: tryParseDate(process.env[APIFY_ENV_VARS.STARTED_AT]),
- timeoutAt: tryParseDate(process.env[APIFY_ENV_VARS.TIMEOUT_AT]),
- defaultKeyValueStoreId: process.env[APIFY_ENV_VARS.DEFAULT_KEY_VALUE_STORE_ID],
- // TODO: defaultStore object ?
- };
+export const getInput = (callback = null) => {
+ return apifyClient.keyValueStores.getRecord({
+ storeId: getDefaultStoreIdOrThrow(),
+ promise: getPromisesDependency(),
+ }, callback);
};
-export const getInput = (callback) => {
- return null;
+/**
+ * Sets output data for the current act run.
+ * This data is stored in the key-value store created specifically for this run,
+ * whose ID is defined in the `APIFY_DEFAULT_KEY_VALUE_STORE_ID` environment variable,
+ * under record with the `OUTPUT` key.
+ * The function has no result.
+ * @param output
+ * @param callback Optional callback.
+ * @return Returns a promise if no callback was provided, otherwise the return value is not defined.
+ */
+export const setOutput = (output, callback = null) => {
+ if (!output) throw new Error('The "output" parameter must be provided.');
+
+ return apifyClient.keyValueStores.setRecord({
+ storeId: getDefaultStoreIdOrThrow(),
+ promise: getPromisesDependency(),
+ body: output.body,
+ contentType: output.contentType,
+ }, callback);
};
-export const setOutput = (callback) => {
- return null;
+/**
+ * Generates a context object which contains meta-data about this act run such as:
+ * ```javascript
+ * {
+ * internalPort: Number,
+ * actId: String,
+ * actRunId: String,
+ * startedAt: Date,
+ * timeoutAt: Date,
+ * defaultKeyValueStoreId: String,
+ * input: {
+ * body: String/Buffer,
+ * contentType: String,
+ * }
+ * }
+ * ```
+ * The information is generate using the APIFY_XXX environment variables and the input data is fetched from Apifier API.
+ * If some of the values cannot be determined, it will be set to null;
+ * we don't throw error in such a case in order to simplify local development and debugging of acts.
+ * @param callback Optional callback.
+ * @return Returns a promise if no callback was provided, otherwise the return value is not defined.
+ */
+export const getContext = (callback = null) => {
+ const context = {
+ internalPort: process.env[APIFY_ENV_VARS.INTERNAL_PORT] || null,
+ actId: process.env[APIFY_ENV_VARS.ACT_ID] || null,
+ actRunId: process.env[APIFY_ENV_VARS.ACT_RUN_ID] || null,
+ startedAt: tryParseDate(process.env[APIFY_ENV_VARS.STARTED_AT]) || null,
+ timeoutAt: tryParseDate(process.env[APIFY_ENV_VARS.TIMEOUT_AT]) || null,
+ defaultKeyValueStoreId: process.env[APIFY_ENV_VARS.DEFAULT_KEY_VALUE_STORE_ID] || null,
+ input: null,
+ };
+
+ return newPromise()
+ .then(() => {
+ if (!context.defaultKeyValueStoreId) return null;
+ return getInput();
+ })
+ .then((input) => {
+ context.input = input || null;
+ return context;
+ })
+ .catch((err) => {
+ if (!callback) throw err;
+ callback(err);
+ });
};
-const exitError = (err, exitCode, message) => {
+const exitWithError = (err, exitCode, message) => {
console.error(message);
console.error(err.stack || err);
process.exit(exitCode);
@@ -52,48 +124,39 @@ export const main = (userFunc) => {
throw new Error('Handler function must be provided as a parameter');
}
- const options = {
- context: getContext(),
- input: null,
- };
-
try {
newPromise()
.then(() => {
- // Read input from the key-value store
- return getInput();
- })
- .then((input) => {
- options.input = input;
+ return getContext();
})
.catch((err) => {
- exitError(err, EXIT_CODES.ERROR_GETTING_INPUT, 'Failed to fetch act input');
+ exitWithError(err, EXIT_CODES.ERROR_GETTING_INPUT, 'Failed to fetch act input');
})
- .then(() => {
- return userFunc(options);
+ .then((context) => {
+ return userFunc(context);
})
.catch((err) => {
- exitError(err, EXIT_CODES.ERROR_USER_FUNCTION_THREW, 'User function threw an exception');
+ exitWithError(err, EXIT_CODES.ERROR_USER_FUNCTION_THREW, 'User function threw an exception');
})
- .then((returnValue) => {
+ .then((userReturnValue) => {
// Save output to the key-value store
- if (returnValue) {
- const obj = {
- json: JSON.stringify(returnValue),
+ if (userReturnValue) {
+ const output = {
+ body: JSON.stringify(userReturnValue),
contentType: 'application/json',
};
- return setOutput(obj);
+ return setOutput(output);
}
})
.catch((err) => {
- exitError(err, EXIT_CODES.ERROR_SETTING_OUTPUT, 'Failed to save act output');
+ exitWithError(err, EXIT_CODES.ERROR_SETTING_OUTPUT, 'Failed to save act output');
})
.then(() => {
process.exit(EXIT_CODES.SUCCESS);
});
} catch (err) {
// This can happen e.g. if there's no Promise dependency
- exitError(err, EXIT_CODES.ERROR_UNKNOWN, 'Unknown error occurred');
+ exitWithError(err, EXIT_CODES.ERROR_UNKNOWN, 'Unknown error occurred');
}
};
diff --git a/src/constants.js b/src/constants.js
index 93880c365..7b2178a13 100644
--- a/src/constants.js
+++ b/src/constants.js
@@ -1,13 +1,13 @@
/**
- * Process exit codes for the act process.
+ * Exit codes for the act process.
*/
export const EXIT_CODES = {
SUCCESS: 0,
- ERROR_USER_FUNCTION_THREW: 1,
- ERROR_GETTING_INPUT: 2,
- ERROR_SETTING_OUTPUT: 3,
- ERROR_UNKNOWN: 4,
+ ERROR_USER_FUNCTION_THREW: 1001,
+ ERROR_GETTING_INPUT: 1002,
+ ERROR_SETTING_OUTPUT: 1003,
+ ERROR_UNKNOWN: 1004,
};
/**