Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Rework timeouts, fix some hanging tests
Ondra Urban637237ae48bb
diff --git a/package.json b/package.jsonindex 9e526f45c..69726bcf7 100644--- a/package.json+++ b/package.json@@ -41,7 +41,7 @@"build": "rm -rf ./build && babel src --out-dir build","build-docs": "npm run clean && npm run build && node ./website/tools/build_docs.js","build-readme": "node ./tools/build_readme.js",- "test": "npm run build && nyc --reporter=html --reporter=text mocha --timeout 60000 --compilers js:babel-core/register --recursive --exit",+ "test": "npm run build && nyc --reporter=html --reporter=text mocha --timeout 60000 --compilers js:babel-core/register --recursive","prepare": "npm run build","prepublishOnly": "(test $RUNNING_FROM_SCRIPT || (echo \"You must use publish.sh instead of 'npm publish' directly!\"; exit 1)) && npm test && npm run lint","clean": "rm -rf build",diff --git a/src/cheerio_crawler.js b/src/cheerio_crawler.jsindex f2aebeec1..47a9f2d8c 100644--- a/src/cheerio_crawler.js+++ b/src/cheerio_crawler.js@@ -6,7 +6,7 @@ import contentType from 'content-type';import log from 'apify-shared/log';import { checkParamOrThrow } from 'apify-client/build/utils';import BasicCrawler from './basic_crawler';-import { createTimeoutPromise } from './utils';+import { addTimeoutToPromise } from './utils';import { getApifyProxyUrl } from './actor';const DEFAULT_OPTIONS = {@@ -318,8 +318,7 @@ class CheerioCrawler {// rejectOnAbortPromise rejects when .abort() is called or BasicCrawler throws.// All running pages are therefore terminated with an error to be reclaimed and retried.const response = await Promise.race([- rfPromise,- createTimeoutPromise(this.requestTimeoutMillis, 'CheerioCrawler: requestFunction timed out.'),+ addTimeoutToPromise(rfPromise, this.requestTimeoutMillis, 'CheerioCrawler: requestFunction timed out.'),this.rejectOnAbortPromise,]);@@ -331,8 +330,11 @@ class CheerioCrawler {const $ = cheerio.load(html);await Promise.race([- this.handlePageFunction({ $, html, request, response }),- createTimeoutPromise(this.handlePageTimeoutMillis, 'CheerioCrawler: handlePageFunction timed out.'),+ addTimeoutToPromise(+ this.handlePageFunction({ $, html, request, response }),+ this.handlePageTimeoutMillis,+ 'CheerioCrawler: handlePageFunction timed out.',+ ),this.rejectOnAbortPromise,]);}diff --git a/src/puppeteer_crawler.js b/src/puppeteer_crawler.jsindex a9433e5a0..c430e0a4e 100644--- a/src/puppeteer_crawler.js+++ b/src/puppeteer_crawler.js@@ -3,7 +3,7 @@ import log from 'apify-shared/log';import _ from 'underscore';import BasicCrawler from './basic_crawler';import PuppeteerPool from './puppeteer_pool';-import { createTimeoutPromise } from './utils';+import { addTimeoutToPromise } from './utils';const DEFAULT_OPTIONS = {gotoFunction: async ({ request, page }) => page.goto(request.url, { timeout: 60000 }),@@ -234,7 +234,7 @@ class PuppeteerCrawler {this.gotoFunction = gotoFunction;if (pageOpsTimeoutMillis) log.warning('options.pageOpsTimeoutMillis is deprecated, use options.handlePageTimeoutSecs instead.');- this.handlePageTimeoutSecs = handlePageTimeoutSecs || Math.ceil(pageOpsTimeoutMillis / 1000);+ this.handlePageTimeoutMillis = handlePageTimeoutSecs * 1000 || pageOpsTimeoutMillis;this.puppeteerPoolOptions = {maxOpenPagesPerInstance,@@ -246,7 +246,7 @@ class PuppeteerCrawler {launchPuppeteerOptions,};- this.puppeteerPool = new PuppeteerPool(this.puppeteerPoolOptions);+ this.puppeteerPool = null; // Constructed when .run()this.basicCrawler = new BasicCrawler({// Basic crawler options.@@ -311,19 +311,18 @@ class PuppeteerCrawler {try {const pageOperationsPromise = this.gotoFunction({ page, request, puppeteerPool: this.puppeteerPool })- .then((response) => {- return Promise.race([- this.handlePageFunction({ page, request, puppeteerPool: this.puppeteerPool, response }),- createTimeoutPromise(this.handlePageTimeoutSecs * 1000, 'PuppeteerCrawler: handlePageFunction timed out.'),- ]);- });+ .then(response => addTimeoutToPromise(+ this.handlePageFunction({ page, request, puppeteerPool: this.puppeteerPool, response }),+ this.handlePageTimeoutMillis,+ 'PuppeteerCrawler: handlePageFunction timed out.',+ ));// rejectOnAbortPromise rejects when .abort() is called or BasicCrawler throws.// All running pages are therefore terminated with an error to be reclaimed and retried.return await Promise.race([pageOperationsPromise, this.rejectOnAbortPromise]);} finally {try {- await Promise.race([page.close(), createTimeoutPromise(PAGE_CLOSE_TIMEOUT_MILLIS, 'Operation timed out.')]);+ await addTimeoutToPromise(page.close(), PAGE_CLOSE_TIMEOUT_MILLIS, 'Operation timed out.');} catch (err) {log.debug('PuppeteerCrawler: Page.close() failed.', { reason: err && err.message });}diff --git a/src/utils.js b/src/utils.jsindex 9b6cc137b..e15b30914 100644--- a/src/utils.js+++ b/src/utils.js@@ -1,7 +1,7 @@-import Promise from 'bluebird';import contentTypeParser from 'content-type';import os from 'os';import fs from 'fs';+import util from 'util';import fsExtra from 'fs-extra';import ApifyClient from 'apify-client';import psTree from '@apify/ps-tree';@@ -28,8 +28,8 @@ const URL_NO_COMMAS_REGEX = XRegExp('https?://(www\\.)?[\\p{L}0-9][-\\p{L}0-9@:%*/const URL_WITH_COMMAS_REGEX = XRegExp('https?://(www\\.)?[\\p{L}0-9][-\\p{L}0-9@:%._\\+~#=]{0,254}[\\p{L}0-9]\\.[a-z]{2,63}(:\\d{1,5})?(/[-\\p{L}0-9@:%_\\+,.~#?&//=\\(\\)]*)?', 'gi'); // eslint-disable-line-const ensureDirPromised = Promise.promisify(fsExtra.ensureDir);-const psTreePromised = Promise.promisify(psTree);+const ensureDirPromised = util.promisify(fsExtra.ensureDir);+const psTreePromised = util.promisify(psTree);/*** Creates an instance of ApifyClient using options as defined in the environment variables.@@ -104,12 +104,12 @@ export const addCharsetToContentType = (contentType) => {let isDockerPromiseCache;const createIsDockerPromise = () => {- const promise1 = Promise+ const promise1 = util.promisify(fs.stat)('/.dockerenv').then(() => true).catch(() => false);- const promise2 = Promise+ const promise2 = util.promisify(fs.readFile)('/proc/self/cgroup', 'utf8').then(content => content.indexOf('docker') !== -1).catch(() => false);@@ -240,7 +240,7 @@ export const getMemoryInfo = async () => {} else {// When running inside Docker container, use container memory limits// This must be promisified here so that we can mock it.- const readPromised = Promise.promisify(fs.readFile);+ const readPromised = util.promisify(fs.readFile);const [totalBytesStr, usedBytesStr] = await Promise.all([readPromised('/sys/fs/cgroup/memory/memory.limit_in_bytes'),@@ -262,7 +262,7 @@ export const getMemoryInfo = async () => {};/**- * Helper function that detrermines if given parameter is an instance of Promise.+ * Helper function that determines if given parameter is an instance of Promise.** @ignore*/@@ -309,6 +309,31 @@ export const getTypicalChromeExecutablePath = () => {}};+/**+ * Wraps the provided Promise with another one that rejects with the given errorMessage+ * after the given timeoutMillis, unless the original promise resolves or rejects earlier.+ *+ * @param {Promise} promise+ * @param {number} timeoutMillis+ * @param {string} errorMessage+ */+export const addTimeoutToPromise = (promise, timeoutMillis, errorMessage) => {+ return new Promise((resolve, reject) => {+ if (!isPromise(promise)) throw new Error('Parameter promise of type Promise must be provided.');+ checkParamOrThrow(timeoutMillis, 'timeoutMillis', 'Number');+ checkParamOrThrow(errorMessage, 'errorMessage', 'String');++ const timeout = setTimeout(() => reject(new Error(errorMessage)), timeoutMillis);+ promise.then((data) => {+ clearTimeout(timeout);+ resolve(data);+ }, (reason) => {+ clearTimeout(timeout);+ reject(reason);+ });+ });+};+/*** Creates a promise that after given time gets rejected with given error.*diff --git a/test/actor.js b/test/actor.jsindex 6ef616e4b..a9a760b7d 100644--- a/test/actor.js+++ b/test/actor.js@@ -4,7 +4,6 @@ import chai, { expect } from 'chai';import chaiAsPromised from 'chai-as-promised';import 'babel-polyfill';import sinon from 'sinon';-import Promise from 'bluebird';import { delayPromise } from 'apify-shared/utilities';import { ENV_VARS, ACT_JOB_STATUSES, LOCAL_ENV_VARS } from 'apify-shared/consts';import { ApifyCallError } from '../build/errors';@@ -38,7 +37,7 @@ const popFreePort = () => freePorts.pop();* Helper function that enables testing of Apify.main()* @returns Promise*/-const testMain = ({ userFunc, exitCode }) => {+const testMain = async ({ userFunc, exitCode }) => {// Mock process.exit() to check exit code and prevent process exitconst processMock = sinon.mock(process);const exitExpectation = processMock@@ -49,47 +48,48 @@ const testMain = ({ userFunc, exitCode }) => {let error = null;- return Promise.resolve()- .then(() => {- return new Promise((resolve, reject) => {- // Invoke main() function, the promise resolves after the user function is run- Apify.main(() => {- try {- // Wait for all tasks in Node.js event loop to finish+ try {+ await Promise.resolve()+ .then(() => {+ return new Promise((resolve, reject) => {+ // Invoke main() function, the promise resolves after the user function is run+ Apify.main(() => {+ try {+ // Wait for all tasks in Node.js event loop to finish+ resolve();+ } catch (err) {+ reject(err);+ return;+ }+ // Call user func to test other behavior (note that it can throw)+ if (userFunc) return userFunc();+ });+ })+ .catch((err) => {+ error = err;+ });+ })+ .then(() => {+ // Waits max 1000 millis for process.exit() mock to be called+ // console.log(`XXX: grand finale: ${err}`);+ return new Promise((resolve) => {+ const waitUntil = Date.now() + 1000;+ const intervalId = setInterval(() => {+ // console.log('test for exitExpectation.called');+ if (!exitExpectation.called && Date.now() < waitUntil) return;+ clearInterval(intervalId);+ // console.log(`exitExpectation.called: ${exitExpectation.called}`);resolve();- } catch (err) {- reject(err);- return;- }- // Call user func to test other behavior (note that it can throw)- if (userFunc) return userFunc();+ }, 10);});})- .catch((err) => {- error = err;- });- })- .then(() => {- // Waits max 1000 millis for process.exit() mock to be called- // console.log(`XXX: grand finale: ${err}`);- return new Promise((resolve) => {- const waitUntil = Date.now() + 1000;- const intervalId = setInterval(() => {- // console.log('test for exitExpectation.called');- if (!exitExpectation.called && Date.now() < waitUntil) return;- clearInterval(intervalId);- // console.log(`exitExpectation.called: ${exitExpectation.called}`);- resolve();- }, 10);+ .then(() => {+ if (error) throw error;+ processMock.verify();});- })- .then(() => {- if (error) throw error;- processMock.verify();- })- .finally(() => {- processMock.restore();- });+ } finally {+ processMock.restore();+ }};diff --git a/test/events.js b/test/events.jsindex 761ec2872..c9d71290c 100644--- a/test/events.js+++ b/test/events.js@@ -1,7 +1,6 @@import 'babel-polyfill';import WebSocket from 'ws';import sinon from 'sinon';-import Promise from 'bluebird';import { expect } from 'chai';import { delayPromise } from 'apify-shared/utilities';import { ENV_VARS } from 'apify-shared/consts';diff --git a/test/puppeteer.js b/test/puppeteer.jsindex c6a383a46..615861525 100644--- a/test/puppeteer.js+++ b/test/puppeteer.js@@ -1,9 +1,9 @@import { expect } from 'chai';import proxy from 'proxy';import http from 'http';+import util from 'util';import portastic from 'portastic';import basicAuthParser from 'basic-auth-parser';-import Promise from 'bluebird';import _ from 'underscore';import sinon from 'sinon';import { ENV_VARS } from 'apify-shared/consts';@@ -59,7 +59,7 @@ after(function () {process.env[ENV_VARS.HEADLESS] = prevEnvHeadless;this.timeout(5 * 1000);- if (proxyServer) return Promise.promisify(proxyServer.close).bind(proxyServer)();+ if (proxyServer) return util.promisify(proxyServer.close).bind(proxyServer)();});@@ -160,7 +160,7 @@ describe('Apify.launchPuppeteer()', () => {});});- it('supports useChrome option', () => {+ it('supports useChrome option', async () => {const mock = sinon.mock(utils);mock.expects('getTypicalChromeExecutablePath').once();@@ -169,27 +169,27 @@ describe('Apify.launchPuppeteer()', () => {useChrome: true,headless: true,};-- return Apify.launchPuppeteer(opts)- .then((result) => {- browser = result;- })- .then(() => {- return browser.newPage();- })- .then((page) => {- return page.content();- })- .then(() => {- return browser.close();- })- .finally(() => {- mock.verify();- mock.restore();- });+ try {+ await Apify.launchPuppeteer(opts)+ .then((result) => {+ browser = result;+ })+ .then(() => {+ return browser.newPage();+ })+ .then((page) => {+ return page.content();+ })+ .then(() => {+ return browser.close();+ });+ } finally {+ mock.verify();+ mock.restore();+ }});- it('should allow to use Apify proxy', () => {+ it('should allow to use Apify proxy', async () => {process.env[ENV_VARS.PROXY_PASSWORD] = 'abc123';process.env[ENV_VARS.PROXY_HOSTNAME] = 'my.host.com';process.env[ENV_VARS.PROXY_PORT] = 123;@@ -205,21 +205,22 @@ describe('Apify.launchPuppeteer()', () => {}).returns(null); // Return null so that it doesn't start proxy-chain- return Apify- .launchPuppeteer({- useApifyProxy: true,- apifyProxySession: 'xxx',- apifyProxyGroups: ['yyy'],- headless: true,- })- .then(browser => browser.close())- .finally(() => {- mock.verify();- mock.restore();- delete process.env[ENV_VARS.PROXY_PASSWORD];- delete process.env[ENV_VARS.PROXY_HOSTNAME];- delete process.env[ENV_VARS.PROXY_PORT];- });+ try {+ await Apify+ .launchPuppeteer({+ useApifyProxy: true,+ apifyProxySession: 'xxx',+ apifyProxyGroups: ['yyy'],+ headless: true,+ })+ .then(browser => browser.close());+ } finally {+ mock.verify();+ mock.restore();+ delete process.env[ENV_VARS.PROXY_PASSWORD];+ delete process.env[ENV_VARS.PROXY_HOSTNAME];+ delete process.env[ENV_VARS.PROXY_PORT];+ }});it('should throw when useApifyProxy=true and proxy password is not set', () => {diff --git a/test/puppeteer_live_view_server.js b/test/puppeteer_live_view_server.jsindex 0ebc3a2c9..af9c3ca08 100644--- a/test/puppeteer_live_view_server.js+++ b/test/puppeteer_live_view_server.js@@ -1,6 +1,5 @@import http from 'http';import { expect, assert } from 'chai';-import Promise from 'bluebird';import puppeteer from 'puppeteer';import { ENV_VARS } from 'apify-shared/consts';import Apify from '../build/index';diff --git a/test/utils.js b/test/utils.jsindex 516ed426d..f1348f95e 100644--- a/test/utils.js+++ b/test/utils.js@@ -4,7 +4,6 @@ import sinon from 'sinon';import fs from 'fs';import path from 'path';import os from 'os';-import Promise from 'bluebird';import requestPromise from 'request-promise-native';import LruCache from 'apify-shared/lru_cache';import { ENV_VARS, LOCAL_ENV_VARS } from 'apify-shared/consts';diff --git a/test/webdriver.js b/test/webdriver.jsindex 91b131f39..77e209a6b 100644--- a/test/webdriver.js+++ b/test/webdriver.js@@ -2,9 +2,9 @@ import _ from 'underscore';import { expect, assert } from 'chai';import proxy from 'proxy';import http from 'http';+import util from 'util';import portastic from 'portastic';import basicAuthParser from 'basic-auth-parser';-import Promise from 'bluebird';// import fs from 'fs';import { processBrowseArgs, getDefaultBrowseOptions } from '../build/webdriver';@@ -54,7 +54,7 @@ before(() => {after(function () {this.timeout(30 * 1000);- if (proxyServer) return Promise.promisify(proxyServer.close).bind(proxyServer)();+ if (proxyServer) return util.promisify(proxyServer.close).bind(proxyServer)();});describe('getDefaultBrowseOptions()', () => {