Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Improved Apify.main() to provide nicer stack traces on errors, fixed unit tests
Jan Curne208977400aa
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 4133bbf73..d7c025ba5 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -8,7 +8,10 @@ xxx- Added `loadedUrl` property to `Request` that contains the final URL of the loaded page after all redirects.- Added memory overload warning log message.- Added `keyValueStore.getPublicUrl` function.-- Updated `DEFAULT_USER_AGENT` and `USER_AGENT_LIST`+- Updated `DEFAULT_USER_AGENT` and `USER_AGENT_LIST` with new User Agents+- Bugfix: `LocalRequestQueue.getRequest()` threw an exception if request was not found+- Added `RequestQueue.getInfo()` function+- Improved `Apify.main()` to provide nicer stack traces on errors0.11.8 / 2019-02-05===================diff --git a/src/actor.js b/src/actor.jsindex 6339ebc7a..9db20569c 100644--- a/src/actor.js+++ b/src/actor.js@@ -219,28 +219,29 @@ export const main = (userFunc) => {// Such a construct is used for testing of actor timeouts and aborts.const intervalId = setInterval(_.noop, 9999999);- try {- newPromise()- .then(() => initializeEvents())- .then(() => userFunc())- .catch((err) => {- stopEvents();- clearInterval(intervalId);- if (!exited) {- exitWithError(err, EXIT_CODES.ERROR_USER_FUNCTION_THREW, 'The function passed to Apify.main() threw an exception:');- }- })- .then(() => {- stopEvents();- clearInterval(intervalId);- if (!exited) {- process.exit(EXIT_CODES.SUCCESS);- }- });- } catch (err) {- // This can happen if, e.g. there's no Promise dependency+ // Using async here to have nice stack traces for errors+ const run = async () => {+ initializeEvents();+ try {+ await userFunc();++ stopEvents();+ clearInterval(intervalId);+ if (!exited) {+ process.exit(EXIT_CODES.SUCCESS);+ }+ } catch (err) {+ stopEvents();+ clearInterval(intervalId);+ if (!exited) {+ exitWithError(err, EXIT_CODES.ERROR_USER_FUNCTION_THREW, 'The function passed to Apify.main() threw an exception:');+ }+ }+ };++ run().catch((err) => {exitWithError(err, EXIT_CODES.ERROR_UNKNOWN, 'Unknown error occurred');- }+ });};let callMemoryWarningIssued = false;diff --git a/test/actor.js b/test/actor.jsindex 5b9c4397c..11cc96cb6 100644--- a/test/actor.js+++ b/test/actor.js@@ -131,6 +131,16 @@ const setEnv = (env) => {};describe('Apify.getEnv()', () => {+ let prevEnv;++ before(() => {+ prevEnv = Apify.getEnv();+ });++ after(() => {+ setEnv(prevEnv);+ });+it('works with null values', () => {const expectedEnv = getEmptyEnv();setEnv(expectedEnv);@@ -218,11 +228,9 @@ describe('Apify.main()', () => {it('on exception in promised user function the process exits with code 91', () => {return testMain({- userFunc: () => {- return delayPromise(20)- .then(() => {- throw new Error('Text exception II');- });+ userFunc: async () => {+ await delayPromise(20);+ throw new Error('Test exception II');},exitCode: 91,});