Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Upgraded to beta apify-client pacakage, fixed lint errors after upgrade of dev packages, minor improvements, bumped package.json version
Jan Curnc052121cfd27
diff --git a/.eslintrc.json b/.eslintrc.jsonindex 5ee33f47e..73d533b23 100644--- a/.eslintrc.json+++ b/.eslintrc.json@@ -13,6 +13,7 @@"no-param-reassign": 0,"consistent-return": 0,"array-callback-return": 0,+ "object-curly-newline": 0,"arrow-body-style": 0,"no-plusplus": 0,"strict": ["error", "global"],diff --git a/package.json b/package.jsonindex ebf84ad04..af869367a 100644--- a/package.json+++ b/package.json@@ -1,6 +1,6 @@{"name": "apify",- "version": "0.4.35",+ "version": "0.4.36","description": "Apify Actor runtime for JavaScript","main": "build/index.js","keywords": [@@ -43,7 +43,7 @@"lint": "npm run build && eslint src test"},"dependencies": {- "apify-client": "^0.1.53",+ "apify-client": "^0.1.55","bluebird": "^3.5.0","content-type": "^1.0.3","proxy-chain": "^0.1.2",diff --git a/src/constants.js b/src/constants.jsindex 7371359ee..60c6e5b35 100644--- a/src/constants.js+++ b/src/constants.js@@ -1,4 +1,6 @@+export const DEFAULT_USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36'; // eslint-disable-line max-len+/*** Exit codes for the act process.* The error codes must be in range 1-128, to avoid collision with signal exits@@ -43,7 +45,7 @@ export const ACT_TASK_STATUSES = {RUNNING: 'RUNNING', // running on workerSUCCEEDED: 'SUCCEEDED', // finished and all goodFAILED: 'FAILED', // run or build failed- TIMING_OUT: 'TIMING-OUT', // timing out now+ TIMING_OUT: 'TIMING-OUT', // timing out nowTIMED_OUT: 'TIMED-OUT', // timed outABORTING: 'ABORTING', // being aborted by userABORTED: 'ABORTED', // aborted by userdiff --git a/src/puppeteer.js b/src/puppeteer.jsindex ecea42860..6d8a09f4b 100644--- a/src/puppeteer.js+++ b/src/puppeteer.js@@ -1,14 +1,11 @@import { checkParamOrThrow } from 'apify-client/build/utils';import { anonymizeProxy, closeAnonymizedProxy } from 'proxy-chain';import { newPromise } from './utils';-import { ENV_VARS } from './constants';+import { ENV_VARS, DEFAULT_USER_AGENT } from './constants';/* global process, require */-const DEFAULT_USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36'; // eslint-disable-line max-len--/*** @memberof module:Apify* @functiondiff --git a/src/webdriver.js b/src/webdriver.jsindex 1bb2c76c8..610d5ba59 100644--- a/src/webdriver.js+++ b/src/webdriver.js@@ -38,7 +38,6 @@ export const getDefaultBrowseOptions = () => {* @ignore*/export class Browser {-constructor(options) {this.options = Object.assign(getDefaultBrowseOptions(), options);diff --git a/test/actor.js b/test/actor.jsindex 1563e88c6..04373f4e8 100644--- a/test/actor.js+++ b/test/actor.js@@ -47,7 +47,9 @@ const testWatchFileWillBecomeEmpty = (path, waitMillis) => {const intervalId = setInterval(() => {const stat = fs.statSync(path);if (stat.size !== 0) {- if (Date.now() - startedAt >= waitMillis) reject(`Watch file not written in ${waitMillis} millis`);+ if (Date.now() - startedAt >= waitMillis) {+ reject(`Watch file not written in ${waitMillis} millis`); // eslint-disable-line prefer-promise-reject-errors+ }} else {clearInterval(intervalId);resolve();@@ -158,9 +160,9 @@ const testMain = ({ userFunc, exitCode }) => {if (userFunc) return userFunc();});})- .catch((err) => {- error = err;- });+ .catch((err) => {+ error = err;+ });}).then(() => {// Waits max 1000 millis for process.exit() mock to be called@@ -280,9 +282,9 @@ describe('Apify.main()', () => {env: {},exitCode: 0,})- .then(() => {- expect(called).to.eql(true);- });+ .then(() => {+ expect(called).to.eql(true);+ });});it('on exception in simple user function the process exits with code 91', () => {@@ -301,9 +303,9 @@ describe('Apify.main()', () => {return new Promise((resolve) => {setTimeout(resolve, 20);})- .then(() => {- throw new Error('Text exception II');- });+ .then(() => {+ throw new Error('Text exception II');+ });},env: {},exitCode: 91,@@ -392,12 +394,12 @@ describe('Apify.getValue()', () => {resolve(input);});})- .then(() => {- assert.fail();- })- .catch((err) => {- expect(err.message).to.be.eql('Test error');- });+ .then(() => {+ assert.fail();+ })+ .catch((err) => {+ expect(err.message).to.be.eql('Test error');+ });}).then(() => {mock.verify();@@ -500,12 +502,12 @@ describe('Apify.getValue()', () => {resolve(input);});})- .then(() => {- assert.fail();- })- .catch((err) => {- expect(err.message).to.contain('The directory does not exist');- });+ .then(() => {+ assert.fail();+ })+ .catch((err) => {+ expect(err.message).to.contain('The directory does not exist');+ });}).then(() => {// Test file instead of dir@@ -673,12 +675,12 @@ describe('Apify.setValue()', () => {resolve();});})- .then(() => {- assert.fail();- })- .catch((err) => {- expect(err.message).to.be.eql('Test error');- });+ .then(() => {+ assert.fail();+ })+ .catch((err) => {+ expect(err.message).to.be.eql('Test error');+ });}).then(() => {mock.verify();@@ -917,12 +919,12 @@ describe('Apify.setValue()', () => {resolve(input);});})- .then(() => {- assert.fail();- })- .catch((err) => {- expect(err.message).to.contain('ENOENT');- });+ .then(() => {+ assert.fail();+ })+ .catch((err) => {+ expect(err.message).to.contain('ENOENT');+ });}).finally(() => {delete process.env.APIFY_DEV_KEY_VALUE_STORE_DIR;@@ -942,9 +944,9 @@ describe('Apify.events', () => {reject(e);}})- .then((arg) => {- expect(arg).to.eql('test event');- });+ .then((arg) => {+ expect(arg).to.eql('test event');+ });});});diff --git a/test/puppeteer.js b/test/puppeteer.jsindex 556d1b8b0..2f4a4b8a6 100644--- a/test/puppeteer.js+++ b/test/puppeteer.js@@ -111,22 +111,22 @@ describe('Apify.launchPuppeteer()', () => {headless: true,proxyUrl: `http://username:[email protected]:${proxyPort}`,})- .then((createdBrowser) => {- browser = createdBrowser;+ .then((createdBrowser) => {+ browser = createdBrowser;- return browser.newPage();- })- .then((openedPage) => {- page = openedPage;+ return browser.newPage();+ })+ .then((openedPage) => {+ page = openedPage;- return page.goto('https://example.com');- })- .then(() => {- expect(wasProxyCalled).to.eql(true);+ return page.goto('https://example.com');+ })+ .then(() => {+ expect(wasProxyCalled).to.eql(true);- return page.content();- })- .then(html => expect(html).to.include('<h1>Example Domain</h1>'))- .then(() => browser.close());+ return page.content();+ })+ .then(html => expect(html).to.include('<h1>Example Domain</h1>'))+ .then(() => browser.close());});});