Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Upgraded to beta apify-client pacakage, fixed lint errors after upgrade of dev packages, minor improvements, bumped package.json version

Jan Curnc052121cfd27
diff --git a/.eslintrc.json b/.eslintrc.json
index 5ee33f47e..73d533b23 100644
--- a/.eslintrc.json
+++ b/.eslintrc.json
@@ -13,6 +13,7 @@
"no-param-reassign": 0,
"consistent-return": 0,
"array-callback-return": 0,
+ "object-curly-newline": 0,
"arrow-body-style": 0,
"no-plusplus": 0,
"strict": ["error", "global"],
diff --git a/package.json b/package.json
index ebf84ad04..af869367a 100644
--- a/package.json
+++ b/package.json
@@ -1,6 +1,6 @@
{
"name": "apify",
- "version": "0.4.35",
+ "version": "0.4.36",
"description": "Apify Actor runtime for JavaScript",
"main": "build/index.js",
"keywords": [
@@ -43,7 +43,7 @@
"lint": "npm run build && eslint src test"
},
"dependencies": {
- "apify-client": "^0.1.53",
+ "apify-client": "^0.1.55",
"bluebird": "^3.5.0",
"content-type": "^1.0.3",
"proxy-chain": "^0.1.2",
diff --git a/src/constants.js b/src/constants.js
index 7371359ee..60c6e5b35 100644
--- a/src/constants.js
+++ b/src/constants.js
@@ -1,4 +1,6 @@
+export const DEFAULT_USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36'; // eslint-disable-line max-len
+
/**
* Exit codes for the act process.
* The error codes must be in range 1-128, to avoid collision with signal exits
@@ -43,7 +45,7 @@ export const ACT_TASK_STATUSES = {
RUNNING: 'RUNNING', // running on worker
SUCCEEDED: 'SUCCEEDED', // finished and all good
FAILED: 'FAILED', // run or build failed
- TIMING_OUT: 'TIMING-OUT', // timing out now
+ TIMING_OUT: 'TIMING-OUT', // timing out now
TIMED_OUT: 'TIMED-OUT', // timed out
ABORTING: 'ABORTING', // being aborted by user
ABORTED: 'ABORTED', // aborted by user
diff --git a/src/puppeteer.js b/src/puppeteer.js
index ecea42860..6d8a09f4b 100644
--- a/src/puppeteer.js
+++ b/src/puppeteer.js
@@ -1,14 +1,11 @@
import { checkParamOrThrow } from 'apify-client/build/utils';
import { anonymizeProxy, closeAnonymizedProxy } from 'proxy-chain';
import { newPromise } from './utils';
-import { ENV_VARS } from './constants';
+import { ENV_VARS, DEFAULT_USER_AGENT } from './constants';
/* global process, require */
-const DEFAULT_USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36'; // eslint-disable-line max-len
-
-
/**
* @memberof module:Apify
* @function
diff --git a/src/webdriver.js b/src/webdriver.js
index 1bb2c76c8..610d5ba59 100644
--- a/src/webdriver.js
+++ b/src/webdriver.js
@@ -38,7 +38,6 @@ export const getDefaultBrowseOptions = () => {
* @ignore
*/
export class Browser {
-
constructor(options) {
this.options = Object.assign(getDefaultBrowseOptions(), options);
diff --git a/test/actor.js b/test/actor.js
index 1563e88c6..04373f4e8 100644
--- a/test/actor.js
+++ b/test/actor.js
@@ -47,7 +47,9 @@ const testWatchFileWillBecomeEmpty = (path, waitMillis) => {
const intervalId = setInterval(() => {
const stat = fs.statSync(path);
if (stat.size !== 0) {
- if (Date.now() - startedAt >= waitMillis) reject(`Watch file not written in ${waitMillis} millis`);
+ if (Date.now() - startedAt >= waitMillis) {
+ reject(`Watch file not written in ${waitMillis} millis`); // eslint-disable-line prefer-promise-reject-errors
+ }
} else {
clearInterval(intervalId);
resolve();
@@ -158,9 +160,9 @@ const testMain = ({ userFunc, exitCode }) => {
if (userFunc) return userFunc();
});
})
- .catch((err) => {
- error = err;
- });
+ .catch((err) => {
+ error = err;
+ });
})
.then(() => {
// Waits max 1000 millis for process.exit() mock to be called
@@ -280,9 +282,9 @@ describe('Apify.main()', () => {
env: {},
exitCode: 0,
})
- .then(() => {
- expect(called).to.eql(true);
- });
+ .then(() => {
+ expect(called).to.eql(true);
+ });
});
it('on exception in simple user function the process exits with code 91', () => {
@@ -301,9 +303,9 @@ describe('Apify.main()', () => {
return new Promise((resolve) => {
setTimeout(resolve, 20);
})
- .then(() => {
- throw new Error('Text exception II');
- });
+ .then(() => {
+ throw new Error('Text exception II');
+ });
},
env: {},
exitCode: 91,
@@ -392,12 +394,12 @@ describe('Apify.getValue()', () => {
resolve(input);
});
})
- .then(() => {
- assert.fail();
- })
- .catch((err) => {
- expect(err.message).to.be.eql('Test error');
- });
+ .then(() => {
+ assert.fail();
+ })
+ .catch((err) => {
+ expect(err.message).to.be.eql('Test error');
+ });
})
.then(() => {
mock.verify();
@@ -500,12 +502,12 @@ describe('Apify.getValue()', () => {
resolve(input);
});
})
- .then(() => {
- assert.fail();
- })
- .catch((err) => {
- expect(err.message).to.contain('The directory does not exist');
- });
+ .then(() => {
+ assert.fail();
+ })
+ .catch((err) => {
+ expect(err.message).to.contain('The directory does not exist');
+ });
})
.then(() => {
// Test file instead of dir
@@ -673,12 +675,12 @@ describe('Apify.setValue()', () => {
resolve();
});
})
- .then(() => {
- assert.fail();
- })
- .catch((err) => {
- expect(err.message).to.be.eql('Test error');
- });
+ .then(() => {
+ assert.fail();
+ })
+ .catch((err) => {
+ expect(err.message).to.be.eql('Test error');
+ });
})
.then(() => {
mock.verify();
@@ -917,12 +919,12 @@ describe('Apify.setValue()', () => {
resolve(input);
});
})
- .then(() => {
- assert.fail();
- })
- .catch((err) => {
- expect(err.message).to.contain('ENOENT');
- });
+ .then(() => {
+ assert.fail();
+ })
+ .catch((err) => {
+ expect(err.message).to.contain('ENOENT');
+ });
})
.finally(() => {
delete process.env.APIFY_DEV_KEY_VALUE_STORE_DIR;
@@ -942,9 +944,9 @@ describe('Apify.events', () => {
reject(e);
}
})
- .then((arg) => {
- expect(arg).to.eql('test event');
- });
+ .then((arg) => {
+ expect(arg).to.eql('test event');
+ });
});
});
diff --git a/test/puppeteer.js b/test/puppeteer.js
index 556d1b8b0..2f4a4b8a6 100644
--- a/test/puppeteer.js
+++ b/test/puppeteer.js
@@ -111,22 +111,22 @@ describe('Apify.launchPuppeteer()', () => {
headless: true,
proxyUrl: `http://username:[email protected]:${proxyPort}`,
})
- .then((createdBrowser) => {
- browser = createdBrowser;
+ .then((createdBrowser) => {
+ browser = createdBrowser;
- return browser.newPage();
- })
- .then((openedPage) => {
- page = openedPage;
+ return browser.newPage();
+ })
+ .then((openedPage) => {
+ page = openedPage;
- return page.goto('https://example.com');
- })
- .then(() => {
- expect(wasProxyCalled).to.eql(true);
+ return page.goto('https://example.com');
+ })
+ .then(() => {
+ expect(wasProxyCalled).to.eql(true);
- return page.content();
- })
- .then(html => expect(html).to.include('<h1>Example Domain</h1>'))
- .then(() => browser.close());
+ return page.content();
+ })
+ .then(html => expect(html).to.include('<h1>Example Domain</h1>'))
+ .then(() => browser.close());
});
});