Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Use shared ESLint config and resolve issues
Ondra Urban6798066ea6bf
diff --git a/.eslintrc.json b/.eslintrc.jsonindex 5f2234a02..a0c8140b4 100644--- a/.eslintrc.json+++ b/.eslintrc.json@@ -1,29 +1,3 @@{- "extends": "airbnb-base",- "plugins": [- "import",- "promise"- ],- "rules": {- "indent": ["error", 4],- "no-underscore-dangle": [2, {- "allow": ["_id"],- "allowAfterThis": true- }],- "no-use-before-define": 0,- "no-param-reassign": 0,- "consistent-return": 0,- "array-callback-return": 0,- "object-curly-newline": 0,- "arrow-body-style": 0,- "no-plusplus": 0,- "strict": ["error", "global"],- "max-len": ["error", 150],- "func-names": 0,- "import/prefer-default-export": 0,- "import/no-absolute-path": 0,- "import/no-extraneous-dependencies": ["error", { "devDependencies": ["**/test/**/*.js"] }],- "no-await-in-loop": 0,- "no-restricted-syntax": ["error", "ForInStatement", "LabeledStatement", "WithStatement"]- }+ "extends": ["@apify"]}diff --git a/package.json b/package.jsonindex 1908b40d3..12481ec47 100644--- a/package.json+++ b/package.json@@ -53,25 +53,26 @@"@apify/http-request": "^1.0.1","@apify/ps-tree": "^1.1.3","apify-client": "^0.5.22",- "apify-shared": "^0.1.51",+ "apify-shared": "^0.1.55","cheerio": "^1.0.0-rc.2","content-type": "^1.0.4",- "express": "^4.16.4",- "fs-extra": "^8.0.1",+ "express": "^4.17.1",+ "fs-extra": "^8.1.0","htmlparser2": "^3.10.1",- "jquery": "^3.3.1",+ "jquery": "^3.4.1","mime": "^2.3.1",- "proxy-chain": "^0.2.5",+ "proxy-chain": "^0.2.7","request": "^2.88.0","request-promise-native": "^1.0.5","rimraf": "^2.6.2","semver": "^6.3.0","socket.io": "^2.2.0","underscore": "^1.9.1",- "ws": "^6.1.2",+ "ws": "^6.2.1","xregexp": "^4.2.4"},"devDependencies": {+ "@apify/eslint-config": "0.0.2","@babel/cli": "^7.0.0","@babel/core": "^7.0.0","@babel/preset-env": "^7.0.0",@@ -83,13 +84,7 @@"chai": "^4.2.0","chai-as-promised": "^7.1.1","chai-subset": "^1.6.0",- "eslint": "^5.9.0",- "eslint-config-airbnb": "^17.1.0",- "eslint-config-airbnb-base": "^13.1.0",- "eslint-plugin-import": "^2.14.0",- "eslint-plugin-jsx-a11y": "^6.1.2",- "eslint-plugin-promise": "^4.0.1",- "eslint-plugin-react": "^7.11.1",+ "eslint": "^6.1.0","fpcollect": "^1.0.4","fpscanner": "^0.1.5","jsdoc": "^3.5.5",diff --git a/src/actor.js b/src/actor.jsindex 2d292107a..8c45c897d 100644--- a/src/actor.js+++ b/src/actor.js@@ -10,8 +10,6 @@ import { apifyClient, addCharsetToContentType, sleep, snakeCaseToCamelCase, isAtimport { maybeStringify } from './key_value_store';import { ApifyCallError } from './errors';-/* globals process */-const METAMORPH_AFTER_SLEEP_MILLIS = 300000;/**diff --git a/src/enqueue_links/click_elements.js b/src/enqueue_links/click_elements.jsindex 637e91920..5fa905b0d 100644--- a/src/enqueue_links/click_elements.js+++ b/src/enqueue_links/click_elements.js@@ -287,7 +287,6 @@ function createFrameNavigatedHandler(page, requests) {async function preventHistoryNavigation(page) {/* istanbul ignore next */return page.evaluate(() => {- /* global window */window.__originalHistory__ = window.history; // eslint-disable-line no-underscore-dangledelete window.history; // Simple override does not work.window.history = {@@ -422,7 +421,6 @@ async function restoreHistoryNavigationAndSaveCapturedUrls(page, requests) {/* eslint-disable no-shadow *//* istanbul ignore next */const stateHistory = await page.evaluate(() => {- /* global window */const { stateHistory } = window.history;window.history = window.__originalHistory__; // eslint-disable-line no-underscore-danglereturn stateHistory;diff --git a/src/index.js b/src/index.jsindex 0e87d2546..a16c05d11 100644--- a/src/index.js+++ b/src/index.js@@ -23,8 +23,6 @@ import PseudoUrl from './pseudo_url';import LiveViewServer from './live_view/live_view_server';import { requestAsBrowser } from './utils_request';-/* globals module */-// Increase the global limit for event emitter memory leak warnings.EventEmitter.defaultMaxListeners = 50;diff --git a/src/puppeteer.js b/src/puppeteer.jsindex 0419dca64..9f1e8288b 100644--- a/src/puppeteer.js+++ b/src/puppeteer.js@@ -8,9 +8,6 @@ import { getTypicalChromeExecutablePath, isAtHome } from './utils';import { getApifyProxyUrl } from './actor';import applyStealthToBrowser from './stealth/stealth';-/* global process, require */--const LAUNCH_PUPPETEER_LOG_OMIT_OPTS = ['proxyUrl', 'userAgent', 'useApifyProxy', 'apifyProxySession', 'apifyProxyGroups',];diff --git a/src/puppeteer_pool.js b/src/puppeteer_pool.jsindex 415b511f9..841fcc1f9 100644--- a/src/puppeteer_pool.js+++ b/src/puppeteer_pool.js@@ -11,8 +11,6 @@ import { launchPuppeteer } from './puppeteer';import { addTimeoutToPromise } from './utils';import LiveViewServer from './live_view/live_view_server';-/* global process */-const PROCESS_KILL_TIMEOUT_MILLIS = 5000;const PAGE_CLOSE_KILL_TIMEOUT_MILLIS = 1000;@@ -635,12 +633,12 @@ class PuppeteerPool {});const results = (await Promise.all(resultPromises)).filter(i => i);switch (results.length) {- case 0:- return null;- case 1:- return results[0];- default:- throw new Error('PuppeteerPool: Multiple instances of PuppeteerPool found using a single browser instance.');+ case 0:+ return null;+ case 1:+ return results[0];+ default:+ throw new Error('PuppeteerPool: Multiple instances of PuppeteerPool found using a single browser instance.');}}diff --git a/src/puppeteer_utils.js b/src/puppeteer_utils.jsindex e6496ea66..845b840ce 100644--- a/src/puppeteer_utils.js+++ b/src/puppeteer_utils.js@@ -34,7 +34,6 @@ const hideWebDriver = async (page) => {checkParamOrThrow(page, 'page', 'Object');await page.evaluateOnNewDocument(() => {- /* global Navigator, window */var modifiedNavigator; // eslint-disable-line no-vartry {if (Navigator.prototype.hasOwnProperty('webdriver')) { // eslint-disable-line no-prototype-builtins@@ -481,7 +480,6 @@ export const infiniteScroll = async (page, options = {}) => {});const checkFinished = setInterval(() => {- // console.log(resourcesStats)if (resourcesStats.oldRequested === resourcesStats.newRequested) {resourcesStats.matchNumber++;if (resourcesStats.matchNumber >= waitForSecs) {@@ -495,15 +493,11 @@ export const infiniteScroll = async (page, options = {}) => {}// check if timeout has been reachedif (timeoutSecs !== 0 && (Date.now() - startTime) / 1000 > timeoutSecs) {- // console.log("Timeout limit reached, exiting infinite scroll.")clearInterval(checkFinished);finished = true;}}, CHECK_INTERVAL_MILLIS);-- /* global window document */-const doScroll = async () => {/* istanbul ignore next */await page.evaluate(async () => {diff --git a/src/stealth/hiding_tricks.js b/src/stealth/hiding_tricks.jsindex 2e55be73a..5fc6ae01e 100644--- a/src/stealth/hiding_tricks.js+++ b/src/stealth/hiding_tricks.js@@ -1,7 +1,5 @@-/* eslint no-undef: 0 *//* istanbul ignore file */-const hideWebDriver = () => {const newProto = window.navigator.__proto__; // eslint-disable-linedelete newProto.webdriver;@@ -51,7 +49,6 @@ const addLanguage = () => {const emulateWebGL = () => {try {- /* global WebGLRenderingContext */const getParameterOld = WebGLRenderingContext.getParameter;WebGLRenderingContext.prototype.getParameter = function (parameter) {// UNMASKED_VENDOR_WEBGL@@ -85,8 +82,6 @@ const emulateWindowFrame = () => {const addPlugins = () => {function mockPluginsAndMimeTypes() {- /* global MimeType MimeTypeArray PluginArray */-// Disguise custom functions as being nativeconst makeFnsNative = (fns = []) => {const oldCall = Function.prototype.call;diff --git a/src/utils.js b/src/utils.jsindex 61b301685..6e00c939a 100644--- a/src/utils.js+++ b/src/utils.js@@ -18,8 +18,6 @@ import { version as apifyClientVersion } from 'apify-client/package.json';import { version as apifyVersion } from '../package.json';import { USER_AGENT_LIST } from './constants';-/* globals process */-/*** Default regular expression to match URLs in a string that may be plain text, JSON, CSV or other. It supports common URL characters* and does not support URLs containing commas or spaces. The URLs also may contain Unicode letters (not symbols).@@ -320,9 +318,9 @@ export const getFirstKey = (dict) => {*/export const getTypicalChromeExecutablePath = () => {switch (os.platform()) {- case 'darwin': return '/Applications/Google Chrome.app/Contents/MacOS/Google Chrome';- case 'win32': return 'C:\\Program Files (x86)\\Google\\Chrome\\Application\\chrome.exe';- default: return 'google-chrome';+ case 'darwin': return '/Applications/Google Chrome.app/Contents/MacOS/Google Chrome';+ case 'win32': return 'C:\\Program Files (x86)\\Google\\Chrome\\Application\\chrome.exe';+ default: return 'google-chrome';}};diff --git a/test/actor.js b/test/actor.jsindex 6b5e127c4..adfc758b1 100644--- a/test/actor.js+++ b/test/actor.js@@ -11,8 +11,6 @@ const Apify = require('../build/index');const { utils: { log } } = Apify;-/* global process, describe, it */-// TODO: override console.log() to test the error messages (now they are printed to console)/*diff --git a/test/enqueue_links/click_elements.js b/test/enqueue_links/click_elements.jsindex 2839fcb45..9871f16ff 100644--- a/test/enqueue_links/click_elements.js+++ b/test/enqueue_links/click_elements.js@@ -3,8 +3,6 @@ import Apify from '../../build';import { RequestQueue } from '../../build/request_queue';import { clickElements, clickElementsAndInterceptNavigationRequests, isTargetRelevant } from '../../build/enqueue_links/click_elements';-/* global window */-describe('enqueueLinksByClickingElements()', () => {let browser;let page;diff --git a/test/puppeteer_utils.js b/test/puppeteer_utils.jsindex 3f6352532..8247524f8 100644--- a/test/puppeteer_utils.js+++ b/test/puppeteer_utils.js@@ -4,8 +4,6 @@ import Apify from '../build/index';const { utils: { log } } = Apify;-/* global window, document */-describe('Apify.utils.puppeteer', () => {let ll;before(() => {diff --git a/test/utils.js b/test/utils.jsindex ba220837f..bef4e48d8 100644--- a/test/utils.js+++ b/test/utils.js@@ -13,8 +13,6 @@ import { ENV_VARS, LOCAL_ENV_VARS } from 'apify-shared/consts';import * as utils from '../build/utils';import Apify from '../build/index';-/* global process, describe, it */-describe('utils.newClient()', () => {it('reads environment variables correctly', () => {process.env[ENV_VARS.API_BASE_URL] = 'http://www.example.com:1234/path/';diff --git a/test/utils_social.js b/test/utils_social.jsindex 259fd66c0..8cc75135c 100644--- a/test/utils_social.js+++ b/test/utils_social.js@@ -8,8 +8,6 @@ chai.use(chaiAsPromised);const { social } = Apify.utils;-/* global describe, it */-describe('utils.social', () => {describe('emailsFromText()', () => {const testEmailsFromText = (text, phones) => {