Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Use shared ESLint config and resolve issues

Ondra Urban6798066ea6bf
diff --git a/.eslintrc.json b/.eslintrc.json
index 5f2234a02..a0c8140b4 100644
--- a/.eslintrc.json
+++ b/.eslintrc.json
@@ -1,29 +1,3 @@
{
- "extends": "airbnb-base",
- "plugins": [
- "import",
- "promise"
- ],
- "rules": {
- "indent": ["error", 4],
- "no-underscore-dangle": [2, {
- "allow": ["_id"],
- "allowAfterThis": true
- }],
- "no-use-before-define": 0,
- "no-param-reassign": 0,
- "consistent-return": 0,
- "array-callback-return": 0,
- "object-curly-newline": 0,
- "arrow-body-style": 0,
- "no-plusplus": 0,
- "strict": ["error", "global"],
- "max-len": ["error", 150],
- "func-names": 0,
- "import/prefer-default-export": 0,
- "import/no-absolute-path": 0,
- "import/no-extraneous-dependencies": ["error", { "devDependencies": ["**/test/**/*.js"] }],
- "no-await-in-loop": 0,
- "no-restricted-syntax": ["error", "ForInStatement", "LabeledStatement", "WithStatement"]
- }
+ "extends": ["@apify"]
}
diff --git a/package.json b/package.json
index 1908b40d3..12481ec47 100644
--- a/package.json
+++ b/package.json
@@ -53,25 +53,26 @@
"@apify/http-request": "^1.0.1",
"@apify/ps-tree": "^1.1.3",
"apify-client": "^0.5.22",
- "apify-shared": "^0.1.51",
+ "apify-shared": "^0.1.55",
"cheerio": "^1.0.0-rc.2",
"content-type": "^1.0.4",
- "express": "^4.16.4",
- "fs-extra": "^8.0.1",
+ "express": "^4.17.1",
+ "fs-extra": "^8.1.0",
"htmlparser2": "^3.10.1",
- "jquery": "^3.3.1",
+ "jquery": "^3.4.1",
"mime": "^2.3.1",
- "proxy-chain": "^0.2.5",
+ "proxy-chain": "^0.2.7",
"request": "^2.88.0",
"request-promise-native": "^1.0.5",
"rimraf": "^2.6.2",
"semver": "^6.3.0",
"socket.io": "^2.2.0",
"underscore": "^1.9.1",
- "ws": "^6.1.2",
+ "ws": "^6.2.1",
"xregexp": "^4.2.4"
},
"devDependencies": {
+ "@apify/eslint-config": "0.0.2",
"@babel/cli": "^7.0.0",
"@babel/core": "^7.0.0",
"@babel/preset-env": "^7.0.0",
@@ -83,13 +84,7 @@
"chai": "^4.2.0",
"chai-as-promised": "^7.1.1",
"chai-subset": "^1.6.0",
- "eslint": "^5.9.0",
- "eslint-config-airbnb": "^17.1.0",
- "eslint-config-airbnb-base": "^13.1.0",
- "eslint-plugin-import": "^2.14.0",
- "eslint-plugin-jsx-a11y": "^6.1.2",
- "eslint-plugin-promise": "^4.0.1",
- "eslint-plugin-react": "^7.11.1",
+ "eslint": "^6.1.0",
"fpcollect": "^1.0.4",
"fpscanner": "^0.1.5",
"jsdoc": "^3.5.5",
diff --git a/src/actor.js b/src/actor.js
index 2d292107a..8c45c897d 100644
--- a/src/actor.js
+++ b/src/actor.js
@@ -10,8 +10,6 @@ import { apifyClient, addCharsetToContentType, sleep, snakeCaseToCamelCase, isAt
import { maybeStringify } from './key_value_store';
import { ApifyCallError } from './errors';
-/* globals process */
-
const METAMORPH_AFTER_SLEEP_MILLIS = 300000;
/**
diff --git a/src/enqueue_links/click_elements.js b/src/enqueue_links/click_elements.js
index 637e91920..5fa905b0d 100644
--- a/src/enqueue_links/click_elements.js
+++ b/src/enqueue_links/click_elements.js
@@ -287,7 +287,6 @@ function createFrameNavigatedHandler(page, requests) {
async function preventHistoryNavigation(page) {
/* istanbul ignore next */
return page.evaluate(() => {
- /* global window */
window.__originalHistory__ = window.history; // eslint-disable-line no-underscore-dangle
delete window.history; // Simple override does not work.
window.history = {
@@ -422,7 +421,6 @@ async function restoreHistoryNavigationAndSaveCapturedUrls(page, requests) {
/* eslint-disable no-shadow */
/* istanbul ignore next */
const stateHistory = await page.evaluate(() => {
- /* global window */
const { stateHistory } = window.history;
window.history = window.__originalHistory__; // eslint-disable-line no-underscore-dangle
return stateHistory;
diff --git a/src/index.js b/src/index.js
index 0e87d2546..a16c05d11 100644
--- a/src/index.js
+++ b/src/index.js
@@ -23,8 +23,6 @@ import PseudoUrl from './pseudo_url';
import LiveViewServer from './live_view/live_view_server';
import { requestAsBrowser } from './utils_request';
-/* globals module */
-
// Increase the global limit for event emitter memory leak warnings.
EventEmitter.defaultMaxListeners = 50;
diff --git a/src/puppeteer.js b/src/puppeteer.js
index 0419dca64..9f1e8288b 100644
--- a/src/puppeteer.js
+++ b/src/puppeteer.js
@@ -8,9 +8,6 @@ import { getTypicalChromeExecutablePath, isAtHome } from './utils';
import { getApifyProxyUrl } from './actor';
import applyStealthToBrowser from './stealth/stealth';
-/* global process, require */
-
-
const LAUNCH_PUPPETEER_LOG_OMIT_OPTS = [
'proxyUrl', 'userAgent', 'useApifyProxy', 'apifyProxySession', 'apifyProxyGroups',
];
diff --git a/src/puppeteer_pool.js b/src/puppeteer_pool.js
index 415b511f9..841fcc1f9 100644
--- a/src/puppeteer_pool.js
+++ b/src/puppeteer_pool.js
@@ -11,8 +11,6 @@ import { launchPuppeteer } from './puppeteer';
import { addTimeoutToPromise } from './utils';
import LiveViewServer from './live_view/live_view_server';
-/* global process */
-
const PROCESS_KILL_TIMEOUT_MILLIS = 5000;
const PAGE_CLOSE_KILL_TIMEOUT_MILLIS = 1000;
@@ -635,12 +633,12 @@ class PuppeteerPool {
});
const results = (await Promise.all(resultPromises)).filter(i => i);
switch (results.length) {
- case 0:
- return null;
- case 1:
- return results[0];
- default:
- throw new Error('PuppeteerPool: Multiple instances of PuppeteerPool found using a single browser instance.');
+ case 0:
+ return null;
+ case 1:
+ return results[0];
+ default:
+ throw new Error('PuppeteerPool: Multiple instances of PuppeteerPool found using a single browser instance.');
}
}
diff --git a/src/puppeteer_utils.js b/src/puppeteer_utils.js
index e6496ea66..845b840ce 100644
--- a/src/puppeteer_utils.js
+++ b/src/puppeteer_utils.js
@@ -34,7 +34,6 @@ const hideWebDriver = async (page) => {
checkParamOrThrow(page, 'page', 'Object');
await page.evaluateOnNewDocument(() => {
- /* global Navigator, window */
var modifiedNavigator; // eslint-disable-line no-var
try {
if (Navigator.prototype.hasOwnProperty('webdriver')) { // eslint-disable-line no-prototype-builtins
@@ -481,7 +480,6 @@ export const infiniteScroll = async (page, options = {}) => {
});
const checkFinished = setInterval(() => {
- // console.log(resourcesStats)
if (resourcesStats.oldRequested === resourcesStats.newRequested) {
resourcesStats.matchNumber++;
if (resourcesStats.matchNumber >= waitForSecs) {
@@ -495,15 +493,11 @@ export const infiniteScroll = async (page, options = {}) => {
}
// check if timeout has been reached
if (timeoutSecs !== 0 && (Date.now() - startTime) / 1000 > timeoutSecs) {
- // console.log("Timeout limit reached, exiting infinite scroll.")
clearInterval(checkFinished);
finished = true;
}
}, CHECK_INTERVAL_MILLIS);
-
- /* global window document */
-
const doScroll = async () => {
/* istanbul ignore next */
await page.evaluate(async () => {
diff --git a/src/stealth/hiding_tricks.js b/src/stealth/hiding_tricks.js
index 2e55be73a..5fc6ae01e 100644
--- a/src/stealth/hiding_tricks.js
+++ b/src/stealth/hiding_tricks.js
@@ -1,7 +1,5 @@
-/* eslint no-undef: 0 */
/* istanbul ignore file */
-
const hideWebDriver = () => {
const newProto = window.navigator.__proto__; // eslint-disable-line
delete newProto.webdriver;
@@ -51,7 +49,6 @@ const addLanguage = () => {
const emulateWebGL = () => {
try {
- /* global WebGLRenderingContext */
const getParameterOld = WebGLRenderingContext.getParameter;
WebGLRenderingContext.prototype.getParameter = function (parameter) {
// UNMASKED_VENDOR_WEBGL
@@ -85,8 +82,6 @@ const emulateWindowFrame = () => {
const addPlugins = () => {
function mockPluginsAndMimeTypes() {
- /* global MimeType MimeTypeArray PluginArray */
-
// Disguise custom functions as being native
const makeFnsNative = (fns = []) => {
const oldCall = Function.prototype.call;
diff --git a/src/utils.js b/src/utils.js
index 61b301685..6e00c939a 100644
--- a/src/utils.js
+++ b/src/utils.js
@@ -18,8 +18,6 @@ import { version as apifyClientVersion } from 'apify-client/package.json';
import { version as apifyVersion } from '../package.json';
import { USER_AGENT_LIST } from './constants';
-/* globals process */
-
/**
* Default regular expression to match URLs in a string that may be plain text, JSON, CSV or other. It supports common URL characters
* and does not support URLs containing commas or spaces. The URLs also may contain Unicode letters (not symbols).
@@ -320,9 +318,9 @@ export const getFirstKey = (dict) => {
*/
export const getTypicalChromeExecutablePath = () => {
switch (os.platform()) {
- case 'darwin': return '/Applications/Google Chrome.app/Contents/MacOS/Google Chrome';
- case 'win32': return 'C:\\Program Files (x86)\\Google\\Chrome\\Application\\chrome.exe';
- default: return 'google-chrome';
+ case 'darwin': return '/Applications/Google Chrome.app/Contents/MacOS/Google Chrome';
+ case 'win32': return 'C:\\Program Files (x86)\\Google\\Chrome\\Application\\chrome.exe';
+ default: return 'google-chrome';
}
};
diff --git a/test/actor.js b/test/actor.js
index 6b5e127c4..adfc758b1 100644
--- a/test/actor.js
+++ b/test/actor.js
@@ -11,8 +11,6 @@ const Apify = require('../build/index');
const { utils: { log } } = Apify;
-/* global process, describe, it */
-
// TODO: override console.log() to test the error messages (now they are printed to console)
/*
diff --git a/test/enqueue_links/click_elements.js b/test/enqueue_links/click_elements.js
index 2839fcb45..9871f16ff 100644
--- a/test/enqueue_links/click_elements.js
+++ b/test/enqueue_links/click_elements.js
@@ -3,8 +3,6 @@ import Apify from '../../build';
import { RequestQueue } from '../../build/request_queue';
import { clickElements, clickElementsAndInterceptNavigationRequests, isTargetRelevant } from '../../build/enqueue_links/click_elements';
-/* global window */
-
describe('enqueueLinksByClickingElements()', () => {
let browser;
let page;
diff --git a/test/puppeteer_utils.js b/test/puppeteer_utils.js
index 3f6352532..8247524f8 100644
--- a/test/puppeteer_utils.js
+++ b/test/puppeteer_utils.js
@@ -4,8 +4,6 @@ import Apify from '../build/index';
const { utils: { log } } = Apify;
-/* global window, document */
-
describe('Apify.utils.puppeteer', () => {
let ll;
before(() => {
diff --git a/test/utils.js b/test/utils.js
index ba220837f..bef4e48d8 100644
--- a/test/utils.js
+++ b/test/utils.js
@@ -13,8 +13,6 @@ import { ENV_VARS, LOCAL_ENV_VARS } from 'apify-shared/consts';
import * as utils from '../build/utils';
import Apify from '../build/index';
-/* global process, describe, it */
-
describe('utils.newClient()', () => {
it('reads environment variables correctly', () => {
process.env[ENV_VARS.API_BASE_URL] = 'http://www.example.com:1234/path/';
diff --git a/test/utils_social.js b/test/utils_social.js
index 259fd66c0..8cc75135c 100644
--- a/test/utils_social.js
+++ b/test/utils_social.js
@@ -8,8 +8,6 @@ chai.use(chaiAsPromised);
const { social } = Apify.utils;
-/* global describe, it */
-
describe('utils.social', () => {
describe('emailsFromText()', () => {
const testEmailsFromText = (text, phones) => {