Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Merging remote
Marek Trunkat47c8a2535205
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 9b1cae4cf..1bdf2430d 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -1,3 +1,7 @@+xxxxxxxxxxxxxxxxxxxx+====================+- Apify SDK package now prints warning when it's outdated.+0.15.3 / 2019-07-29====================- `Apify.launchPuppeteer()` now accepts `proxyUrl` with the `https`, `socks4`@@ -33,6 +37,7 @@- Updated Puppeteer to 1.18.1- Updated `apify-client` to 0.5.22+0.14.15 / 2019-05-31====================- Fixes in `RequestQueue` to deal with inconsistencies in the underlying data storagediff --git a/package.json b/package.jsonindex 51efd2d7e..73e1a7e37 100644--- a/package.json+++ b/package.json@@ -65,6 +65,7 @@"request": "^2.88.0","request-promise-native": "^1.0.5","rimraf": "^2.6.2",+ "semver": "^6.3.0","socket.io": "^2.2.0","underscore": "^1.9.1","ws": "^6.1.2",diff --git a/src/index.js b/src/index.jsindex 451deba45..3a6df31c9 100644--- a/src/index.js+++ b/src/index.js@@ -15,7 +15,7 @@ import Request from './request';import { RequestList, openRequestList } from './request_list';import { openRequestQueue } from './request_queue';import SettingsRotator from './settings_rotator';-import { apifyClient, getMemoryInfo, isAtHome, publicUtils, logSystemInfo } from './utils';+import { apifyClient, getMemoryInfo, isAtHome, publicUtils, logSystemInfo, printOudatedSdkWarning } from './utils';import { puppeteerUtils } from './puppeteer_utils';import { socialUtils } from './utils_social';import { enqueueLinks } from './enqueue_links/enqueue_links';@@ -40,6 +40,9 @@ if (process.env[EMULATION_ENV_VAR]) {// Logging some basic system info (apify and apify-client version, NodeJS version, ...).logSystemInfo();+// Log warning if SDK is outdated.+printOudatedSdkWarning();+/*** The following section describes all functions and properties provided by the `apify` package,* except individual classes and namespaces that have their separate, detailed, documentation pagesdiff --git a/src/utils.js b/src/utils.jsindex aacf79814..1dac583f0 100644--- a/src/utils.js+++ b/src/utils.js@@ -10,6 +10,7 @@ import requestPromise from 'request-promise-native';import XRegExp from 'xregexp';import cheerio from 'cheerio';import log from 'apify-shared/log';+import semver from 'semver';import { getRandomInt } from 'apify-shared/utilities';import { ENV_VARS, LOCAL_ENV_VARS } from 'apify-shared/consts';import { checkParamOrThrow } from 'apify-client/build/utils';@@ -617,6 +618,20 @@ export const snakeCaseToCamelCase = (snakeCaseStr) => {.join('');};+/**+ * Prints a warning if this version of Apify SDK is outdated.+ *+ * @ignore+ */+export const printOudatedSdkWarning = () => {+ const latestApifyVersion = process.env[ENV_VARS.SDK_LATEST_VERSION];+ if (!latestApifyVersion || !semver.lt(apifyVersion, latestApifyVersion)) return;++ // eslint-disable-next-line+ log.warning(`You are using an outdated version (${apifyVersion}) of Apify SDK. We recommend you to update to the latest version (${latestApifyVersion}).+ Read more about Apify SDK versioning at: https://kb.apify.com/en/articles/3184510-apify-sdk-versioning`);+};+/*** A namespace that contains various utilities.*diff --git a/test/utils.js b/test/utils.jsindex 21653e457..d19209ada 100644--- a/test/utils.js+++ b/test/utils.js@@ -5,8 +5,10 @@ import fs from 'fs';import path from 'path';import os from 'os';import cheerio from 'cheerio';+import semver from 'semver';import requestPromise from 'request-promise-native';import LruCache from 'apify-shared/lru_cache';+import log from 'apify-shared/log';import { ENV_VARS, LOCAL_ENV_VARS } from 'apify-shared/consts';import * as utils from '../build/utils';import Apify from '../build/index';@@ -861,6 +863,7 @@ describe('utils.addTimeoutToPromise()', () => {clock.restore();}});+it('should not timeout too soon', async () => {const clock = sinon.useFakeTimers();try {@@ -878,3 +881,39 @@ describe('utils.addTimeoutToPromise()', () => {}});});++describe('utils.printOudatedSdkWarning()', () => {+ let logMock;++ const currentVersion = require('../package.json').version; // eslint-disable-line++ beforeEach(() => {+ logMock = sinon.mock(log);+ });++ afterEach(() => {+ logMock.verify();+ logMock.restore();+ });++ it('should do nothing when ENV_VARS.SDK_LATEST_VERSION is not set', () => {+ delete process.env[ENV_VARS.SDK_LATEST_VERSION];+ logMock.expects('warning').never();+ utils.printOudatedSdkWarning();+ });++ it('should correctly work when outdated', () => {+ process.env[ENV_VARS.SDK_LATEST_VERSION] = semver.inc(currentVersion, 'minor');+ console.log(process.env[ENV_VARS.SDK_LATEST_VERSION]);+ logMock.expects('warning').once();+ utils.printOudatedSdkWarning();+ delete process.env[ENV_VARS.SDK_LATEST_VERSION];+ });++ it('should correctly work when up to date', () => {+ process.env[ENV_VARS.SDK_LATEST_VERSION] = '0.13.0';+ logMock.expects('warning').never();+ utils.printOudatedSdkWarning();+ delete process.env[ENV_VARS.SDK_LATEST_VERSION];+ });+});