Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Development of better act run input/output

Jan Curne55894af3231
diff --git a/README.md b/README.md
index 1684a7efc..e2cc6f5a6 100644
--- a/README.md
+++ b/README.md
@@ -84,11 +84,11 @@ server.listen(process.env.APIFY_INTERNAL_PORT|0, (err) => {
process.exit(1);
}
console.log('Hey I am ready');
- Apifier.heyIAmReady();
+ Apifier.readyFreddy();
});
```
-Note that by calling `Apifier.heyIAmReady()` you tell the Actor runtime that your server is ready to start
+Note that by calling `Apifier.readyFreddy()` you tell the Actor runtime that your server is ready to start
receiving HTTP requests over the port specified by the `APIFY_INTERNAL_PORT` environment variable.
diff --git a/package.json b/package.json
index e6d9b94b8..8059b01e8 100644
--- a/package.json
+++ b/package.json
@@ -14,7 +14,8 @@
"url": "https://www.apifier.com"
},
"contributors": [
- "Jan Curn <[email protected]>"
+ "Jan Curn <[email protected]>",
+ "Marek Trunkat <[email protected]>"
],
"license": "Apache-2.0",
"repository": {
@@ -37,8 +38,7 @@
"lint": "npm run build && eslint src test"
},
"dependencies": {
- "body-parser": "^1.17.1",
- "express": "^4.15.2",
+ "apify-client": "^0.0.1",
"request": "^2.81.0",
"request-promise": "^4.2.0",
"underscore": "^1.8.3"
diff --git a/src/actor.js b/src/actor.js
index 2109a5b1e..8ce6b4c6b 100644
--- a/src/actor.js
+++ b/src/actor.js
@@ -1,85 +1,113 @@
import fs from 'fs';
-import express from 'express';
-import bodyParser from 'body-parser';
+import { APIFY_ENV_VARS, EXIT_CODES } from './constants';
import { newPromise } from './utils';
/* global process */
+/**
+ * Tries to parse a string with date.
+ * @param str Date string
+ * @return Returns either a Date object or undefined
+ */
+const tryParseDate = (str) => {
+ const unix = Date.parse(str);
+ return unix > 0 ? new Date(unix) : undefined;
+};
+
+/**
+ * Gets a context object which contains meta-data about this act run
+ * extracted from the APIFY_XXX environment variables.
+ * @return Returns a dictionary with all properties that could be determined from environment variables.
+ */
+export const getContext = () => {
+ return {
+ internalPort: process.env[APIFY_ENV_VARS.INTERNAL_PORT],
+ actId: process.env[APIFY_ENV_VARS.ACT_ID],
+ actRunId: process.env[APIFY_ENV_VARS.ACT_RUN_ID],
+ startedAt: tryParseDate(process.env[APIFY_ENV_VARS.STARTED_AT]),
+ timeoutAt: tryParseDate(process.env[APIFY_ENV_VARS.TIMEOUT_AT]),
+ defaultKeyValueStoreId: process.env[APIFY_ENV_VARS.DEFAULT_KEY_VALUE_STORE_ID],
+ // TODO: defaultStore object ?
+ };
+};
+
+export const getInput = (callback) => {
+ return null;
+};
+
+export const setOutput = (callback) => {
+ return null;
+};
+
+
+const exitError = (err, exitCode, message) => {
+ console.error(message);
+ console.error(err.stack || err);
+ process.exit(exitCode);
+};
+
export const main = (userFunc) => {
if (!userFunc || typeof (userFunc) !== 'function') {
throw new Error('Handler function must be provided as a parameter');
}
- const serverPort = parseInt(process.env.APIFY_INTERNAL_PORT, 10);
- if (!(serverPort > 0 && serverPort < 65536)) {
- throw new Error('APIFY_INTERNAL_PORT environment variable must have a value from 1 to 65535.');
- }
- const handler = (req, res) => {
- const options = {
- input: {
- body: req.method === 'GET' ? null : req.body,
- method: req.method,
- contentType: req.headers['content-type'],
- },
- };
+ const options = {
+ context: getContext(),
+ input: null,
+ };
+ try {
newPromise()
.then(() => {
- return userFunc(options);
+ // Read input from the key-value store
+ return getInput();
})
- .then(() => {
- res.statusCode = 200;
+ .then((input) => {
+ options.input = input;
})
.catch((err) => {
- console.error('Main function ended with exception:');
- console.error(err.stack || err);
- res.statusCode = 500;
+ exitError(err, EXIT_CODES.ERROR_GETTING_INPUT, 'Failed to fetch act input');
})
.then(() => {
- res.end((err) => {
- if (err) {
- console.log(`Failed to send HTTP response: ${err}`);
- process.exit(2);
- }
- if (res.statusCode === 500) {
- process.exit(1);
- } else {
- console.log('Main function ended successfully');
- process.exit(0);
- }
- });
+ return userFunc(options);
+ })
+ .catch((err) => {
+ exitError(err, EXIT_CODES.ERROR_USER_FUNCTION_THREW, 'User function threw an exception');
+ })
+ .then((returnValue) => {
+ // Save output to the key-value store
+ if (returnValue) {
+ const obj = {
+ json: JSON.stringify(returnValue),
+ contentType: 'application/json',
+ };
+ return setOutput(obj);
+ }
})
.catch((err) => {
- console.log(`Something went terribly wrong: ${err}`);
- process.exit(3);
+ exitError(err, EXIT_CODES.ERROR_SETTING_OUTPUT, 'Failed to save act output');
+ })
+ .then(() => {
+ process.exit(EXIT_CODES.SUCCESS);
});
- };
-
- const app = express();
-
- // parse JSON, pass texts and raw data
- app.use(bodyParser.json({ type: '*/json' }));
- app.use(bodyParser.text({ type: 'text/*' }));
- app.use(bodyParser.raw({ type: '*/*' }));
- app.use(handler);
-
- // TODO: handle errors!
- app.listen(serverPort, () => {
- // console.log(`Listening on port ${serverPort}`);
- heyIAmReady();
- });
+ } catch (err) {
+ // This can happen e.g. if there's no Promise dependency
+ exitError(err, EXIT_CODES.ERROR_UNKNOWN, 'Unknown error occurred');
+ }
};
/**
* Notifies Apifier runtime that act is listening on port specified by the APIFY_INTERNAL_PORT environment
* variable and is ready to receive a HTTP request with act input.
*/
-export const heyIAmReady = () => {
- const watchFileName = process.env.APIFY_WATCH_FILE;
+export const readyFreddy = () => {
+ const watchFileName = process.env[APIFY_ENV_VARS.WATCH_FILE];
if (watchFileName) {
fs.writeFile(watchFileName, '', (err) => {
if (err) console.log(`WARNING: Cannot write to watch file ${watchFileName}: ${err}`);
});
+ } else {
+ console.log(`WARNING: ${APIFY_ENV_VARS.WATCH_FILE} environment variable not specified, readyFreddy() has no effect.`);
}
};
diff --git a/src/constants.js b/src/constants.js
new file mode 100644
index 000000000..93880c365
--- /dev/null
+++ b/src/constants.js
@@ -0,0 +1,24 @@
+
+/**
+ * Process exit codes for the act process.
+ */
+export const EXIT_CODES = {
+ SUCCESS: 0,
+ ERROR_USER_FUNCTION_THREW: 1,
+ ERROR_GETTING_INPUT: 2,
+ ERROR_SETTING_OUTPUT: 3,
+ ERROR_UNKNOWN: 4,
+};
+
+/**
+ * Dictionary of APIFY_XXX environment variable names.
+ */
+export const APIFY_ENV_VARS = {
+ INTERNAL_PORT: 'APIFY_INTERNAL_PORT',
+ ACT_ID: 'APIFY_ACT_ID',
+ ACT_RUN_ID: 'APIFY_ACT_RUN_ID',
+ STARTED_AT: 'APIFY_STARTED_AT',
+ TIMEOUT_AT: 'APIFY_TIMEOUT_AT',
+ DEFAULT_KEY_VALUE_STORE_ID: 'APIFY_DEFAULT_KEY_VALUE_STORE_ID',
+ WATCH_FILE: 'APIFY_WATCH_FILE',
+};
diff --git a/src/crawler.js b/src/crawler.js
index 6b0c54f9d..a2c8e82d7 100644
--- a/src/crawler.js
+++ b/src/crawler.js
@@ -1,3 +1,5 @@
+/*
+TODO: delete this
import _ from 'underscore';
import { newPromise, objectToQueryString } from './utils';
@@ -70,3 +72,4 @@ export const startCrawler = (crawlerId, params, userId, token) => {
return execution;
});
};
+*/
diff --git a/src/index.js b/src/index.js
index 340a97a28..9a62cf7f6 100644
--- a/src/index.js
+++ b/src/index.js
@@ -1,11 +1,11 @@
import { setPromisesDependency, getPromisesDependency } from './utils';
-import { main, heyIAmReady } from './actor';
+import { main, readyFreddy } from './actor';
import openKeyValueStore from './key-value-store';
import { setDefaultToken, setDefaultUserId, getAllCrawlers, startCrawler } from './crawler';
const Apifier = {
main,
- heyIAmReady,
+ readyFreddy,
setPromisesDependency,
getPromisesDependency,
openKeyValueStore,
diff --git a/test/actor.js b/test/actor.js
index b11bbee9f..b1b2c540d 100644
--- a/test/actor.js
+++ b/test/actor.js
@@ -187,10 +187,10 @@ describe('Apifier.main()', () => {
});
-describe('Apifier.heyIAmReady()', () => {
+describe('Apifier.readyFreddy()', () => {
it('it works as expected', () => {
process.env.APIFY_WATCH_FILE = createWatchFile();
- Apifier.heyIAmReady();
+ Apifier.readyFreddy();
return testWatchFileWillBecomeEmpty(process.env.APIFY_WATCH_FILE, 1000);
});
});