Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

add namespaced loggers (#642)

Paulo Cesarffd44d5ec9fd
diff --git a/package.json b/package.json
index dc49f23eb..2ef32f7c5 100644
--- a/package.json
+++ b/package.json
@@ -109,7 +109,7 @@
"sinon": "^9.0.0",
"sinon-stub-promise": "^4.0.0",
"socket.io-client": "^2.2.0",
- "typescript": "3.7.5",
+ "typescript": "^3.8.3",
"ua-parser-js": "^0.7.20",
"why-is-node-running": "^2.0.3"
},
diff --git a/src/autoscaling/autoscaled_pool.js b/src/autoscaling/autoscaled_pool.js
index 637de6975..aa20b5f81 100644
--- a/src/autoscaling/autoscaled_pool.js
+++ b/src/autoscaling/autoscaled_pool.js
@@ -1,21 +1,8 @@
-import * as _ from 'underscore';
import { betterSetInterval, betterClearInterval } from 'apify-shared/utilities';
import { checkParamOrThrow } from 'apify-client/build/utils';
-import log from '../utils_log';
import Snapshotter, { SnapshotterOptions } from './snapshotter'; // eslint-disable-line import/named,no-unused-vars
import SystemStatus, { SystemStatusOptions } from './system_status'; // eslint-disable-line import/named,no-unused-vars
-
-const DEFAULT_OPTIONS = {
- maxConcurrency: 1000,
- minConcurrency: 1,
- desiredConcurrencyRatio: 0.90,
- scaleUpStepRatio: 0.05,
- scaleDownStepRatio: 0.05,
- maybeRunIntervalSecs: 0.5,
- loggingIntervalSecs: 60,
- autoscaleIntervalSecs: 10,
-};
-
+import defaultLog from '../utils_log';
/**
* @typedef AutoscaledPoolOptions
@@ -127,21 +114,23 @@ class AutoscaledPool {
*/
constructor(options = {}) {
const {
- maxConcurrency,
- minConcurrency,
+ maxConcurrency = 1000,
+ minConcurrency = 1,
desiredConcurrency,
- desiredConcurrencyRatio,
- scaleUpStepRatio,
- scaleDownStepRatio,
- maybeRunIntervalSecs,
- loggingIntervalSecs,
- autoscaleIntervalSecs,
+ desiredConcurrencyRatio = 0.90,
+ scaleUpStepRatio = 0.05,
+ scaleDownStepRatio = 0.05,
+ maybeRunIntervalSecs = 0.5,
+ loggingIntervalSecs = 60,
+ autoscaleIntervalSecs = 10,
runTaskFunction,
isFinishedFunction,
isTaskReadyFunction,
systemStatusOptions,
snapshotterOptions,
- } = _.defaults({}, options, DEFAULT_OPTIONS);
+ log = defaultLog,
+ } = options;
+
checkParamOrThrow(maxConcurrency, 'options.maxConcurrency', 'Number');
checkParamOrThrow(minConcurrency, 'options.minConcurrency', 'Number');
@@ -158,6 +147,8 @@ class AutoscaledPool {
checkParamOrThrow(systemStatusOptions, 'options.systemStatusOptions', 'Maybe Object');
checkParamOrThrow(snapshotterOptions, 'options.snapshotterOptions', 'Maybe Object');
+ this.log = log.child({ prefix: 'AutoscaledPool' });
+
// Configurable properties.
this.desiredConcurrencyRatio = desiredConcurrencyRatio;
this.scaleUpStepRatio = scaleUpStepRatio;
@@ -183,7 +174,7 @@ class AutoscaledPool {
// Create instances with correct options.
const ssoCopy = Object.assign({}, systemStatusOptions);
- if (!ssoCopy.snapshotter) ssoCopy.snapshotter = new Snapshotter(snapshotterOptions);
+ if (!ssoCopy.snapshotter) ssoCopy.snapshotter = new Snapshotter({ ...snapshotterOptions, log: this.log });
this.snapshotter = ssoCopy.snapshotter;
this.systemStatus = new SystemStatus(ssoCopy);
}
@@ -193,7 +184,7 @@ class AutoscaledPool {
* @ignore
*/
setMaxConcurrency(maxConcurrency) {
- log.deprecated('AutoscaledPool.setMaxConcurrency() is deprecated, use the "maxConcurrency" property instead');
+ this.log.deprecated('setMaxConcurrency() is deprecated, use the "maxConcurrency" property instead');
this._maxConcurrency = maxConcurrency;
}
@@ -201,7 +192,7 @@ class AutoscaledPool {
* @ignore
*/
setMinConcurrency(minConcurrency) {
- log.deprecated('AutoscaledPool.setMaxConcurrency() is deprecated, use the "maxConcurrency" property instead');
+ this.log.deprecated('setMaxConcurrency() is deprecated, use the "maxConcurrency" property instead');
this._minConcurrency = minConcurrency;
}
@@ -349,7 +340,7 @@ class AutoscaledPool {
let timeout;
if (timeoutSecs) {
timeout = setTimeout(() => {
- const err = new Error('AutoscaledPool: The pool\'s running tasks did not finish'
+ const err = new Error('The pool\'s running tasks did not finish'
+ `in ${timeoutSecs} secs after pool.pause() invocation.`);
reject(err);
}, timeoutSecs);
@@ -401,7 +392,7 @@ class AutoscaledPool {
const currentStatus = this.systemStatus.getCurrentStatus();
const { isSystemIdle } = currentStatus;
if (!isSystemIdle && this._currentConcurrency >= this._minConcurrency) {
- log.perf('AutoscaledPool: Task will not be run. System is overloaded.', currentStatus);
+ this.log.perf('Task will not be run. System is overloaded.', currentStatus);
return done();
}
// - a task is ready.
@@ -413,7 +404,7 @@ class AutoscaledPool {
// We might have already rejected this promise.
if (this.reject) {
// No need to log all concurrent errors.
- log.exception(err, 'AutoscaledPool: isTaskReadyFunction failed');
+ this.log.exception(err, 'isTaskReadyFunction failed');
this.reject(err);
}
} finally {
@@ -444,7 +435,7 @@ class AutoscaledPool {
// We might have already rejected this promise.
if (this.reject) {
// No need to log all concurrent errors.
- log.exception(err, 'AutoscaledPool: runTaskFunction failed.');
+ this.log.exception(err, 'runTaskFunction failed.');
this.reject(err);
}
}
@@ -486,7 +477,7 @@ class AutoscaledPool {
const now = Date.now();
if (now > this.lastLoggingTime + this.loggingIntervalMillis) {
this.lastLoggingTime = now;
- log.info('AutoscaledPool state', {
+ this.log.info('state', {
currentConcurrency: this._currentConcurrency,
desiredConcurrency: this._desiredConcurrency,
systemStatus,
@@ -508,7 +499,7 @@ class AutoscaledPool {
_scaleUp(systemStatus) {
const step = Math.ceil(this._desiredConcurrency * this.scaleUpStepRatio);
this._desiredConcurrency = Math.min(this._maxConcurrency, this._desiredConcurrency + step);
- log.debug('AutoscaledPool: scaling up', {
+ this.log.debug('scaling up', {
oldConcurrency: this._desiredConcurrency - step,
newConcurrency: this._desiredConcurrency,
systemStatus,
@@ -525,7 +516,7 @@ class AutoscaledPool {
_scaleDown(systemStatus) {
const step = Math.ceil(this._desiredConcurrency * this.scaleUpStepRatio);
this._desiredConcurrency = Math.max(this._minConcurrency, this._desiredConcurrency - step);
- log.debug('AutoscaledPool: scaling down', {
+ this.log.debug('scaling down', {
oldConcurrency: this._desiredConcurrency + step,
newConcurrency: this._desiredConcurrency,
systemStatus,
@@ -551,7 +542,7 @@ class AutoscaledPool {
} catch (err) {
if (this.reject) {
// No need to log all concurrent errors.
- log.exception(err, 'AutoscaledPool: isFinishedFunction failed.');
+ this.log.exception(err, 'isFinishedFunction failed.');
this.reject(err);
}
} finally {
diff --git a/src/autoscaling/snapshotter.js b/src/autoscaling/snapshotter.js
index f87b2cab2..df72d1ced 100644
--- a/src/autoscaling/snapshotter.js
+++ b/src/autoscaling/snapshotter.js
@@ -1,23 +1,11 @@
import * as os from 'os';
-import * as _ from 'underscore';
import { betterSetInterval, betterClearInterval } from 'apify-shared/utilities';
import { ACTOR_EVENT_NAMES, ENV_VARS } from 'apify-shared/consts';
import { checkParamOrThrow } from 'apify-client/build/utils';
import { getMemoryInfo, isAtHome, apifyClient } from '../utils';
-import log from '../utils_log';
import events from '../events';
+import defaultLog from '../utils_log';
-const DEFAULT_OPTIONS = {
- eventLoopSnapshotIntervalSecs: 0.5,
- maxBlockedMillis: 50, // 0.05
- memorySnapshotIntervalSecs: 1,
- clientSnapshotIntervalSecs: 1,
- cpuSnapshotIntervalSecs: 1,
- maxUsedMemoryRatio: 0.7,
- maxUsedCpuRatio: 0.95,
- snapshotHistorySecs: 30,
- maxClientErrors: 3,
-};
const RESERVE_MEMORY_RATIO = 0.5;
const CLIENT_RATE_LIMIT_ERROR_RETRY_COUNT = 2;
const CRITICAL_OVERLOAD_RATE_LIMIT_MILLIS = 10000;
@@ -86,16 +74,17 @@ class Snapshotter {
*/
constructor(options = {}) {
const {
- eventLoopSnapshotIntervalSecs,
- cpuSnapshotIntervalSecs,
- memorySnapshotIntervalSecs,
- clientSnapshotIntervalSecs,
- snapshotHistorySecs,
- maxBlockedMillis,
- maxUsedMemoryRatio,
- maxUsedCpuRatio,
- maxClientErrors,
- } = _.defaults({}, options, DEFAULT_OPTIONS);
+ eventLoopSnapshotIntervalSecs = 0.5,
+ cpuSnapshotIntervalSecs = 1,
+ memorySnapshotIntervalSecs = 1,
+ clientSnapshotIntervalSecs = 1,
+ snapshotHistorySecs = 30,
+ maxBlockedMillis = 50,
+ maxUsedMemoryRatio = 0.7,
+ maxUsedCpuRatio = 0.95,
+ maxClientErrors = 3,
+ log = defaultLog,
+ } = options;
checkParamOrThrow(eventLoopSnapshotIntervalSecs, 'options.eventLoopSnapshotIntervalSecs', 'Number');
checkParamOrThrow(memorySnapshotIntervalSecs, 'options.memorySnapshotIntervalSecs', 'Number');
@@ -107,6 +96,7 @@ class Snapshotter {
checkParamOrThrow(maxUsedCpuRatio, 'options.maxUsedCpuRatio', 'Number');
checkParamOrThrow(maxClientErrors, 'options.maxClientErrors', 'Number');
+ this.log = log.child({ prefix: 'Snapshotter' });
this.eventLoopSnapshotIntervalMillis = eventLoopSnapshotIntervalSecs * 1000;
this.memorySnapshotIntervalMillis = memorySnapshotIntervalSecs * 1000;
@@ -276,7 +266,7 @@ class Snapshotter {
this.memorySnapshots.push(snapshot);
} catch (err) {
- log.exception(err, 'Snapshotter: Memory snapshot failed.');
+ this.log.exception(err, 'Memory snapshot failed.');
} finally {
intervalCallback();
}
@@ -299,7 +289,7 @@ class Snapshotter {
if (isCriticalOverload) {
const usedPercentage = Math.round((memCurrentBytes / this.maxMemoryBytes) * 100);
const toMb = bytes => Math.round(bytes / (1024 ** 2));
- log.warning('Memory is critically overloaded. '
+ this.log.warning('Memory is critically overloaded. '
+ `Using ${toMb(memCurrentBytes)} MB of ${toMb(this.maxMemoryBytes)} MB (${usedPercentage}%). Consider increasing the actor memory.`);
this.lastLoggedCriticalMemoryOverloadAt = now;
}
@@ -459,7 +449,7 @@ class Snapshotter {
} else {
this.maxMemoryBytes = Math.ceil(totalBytes / 4);
// NOTE: Log as AutoscaledPool, so that users are not confused what "Snapshotter" is
- log.info(`AutoscaledPool: Setting max memory of this run to ${Math.round(this.maxMemoryBytes / 1024 / 1024)} MB. Use the ${ENV_VARS.MEMORY_MBYTES} environment variable to override it.`); // eslint-disable-line max-len
+ this.log.info(`Setting max memory of this run to ${Math.round(this.maxMemoryBytes / 1024 / 1024)} MB. Use the ${ENV_VARS.MEMORY_MBYTES} environment variable to override it.`); // eslint-disable-line max-len
}
}
}
diff --git a/src/autoscaling/system_status.js b/src/autoscaling/system_status.js
index 8ed1dfbc7..69fd2fa94 100644
--- a/src/autoscaling/system_status.js
+++ b/src/autoscaling/system_status.js
@@ -1,16 +1,7 @@
import { checkParamOrThrow } from 'apify-client/build/utils';
-import * as _ from 'underscore';
import Snapshotter from './snapshotter'; // eslint-disable-line import/no-duplicates
import { weightedAvg } from '../utils';
-const DEFAULT_OPTIONS = {
- currentHistorySecs: 5,
- maxMemoryOverloadedRatio: 0.2,
- maxEventLoopOverloadedRatio: 0.4,
- maxCpuOverloadedRatio: 0.4,
- maxClientOverloadedRatio: 0.3,
-};
-
// TODO yin: Add `@property clientInfo` as in `SystemStatus._isSystemIdle()`
/**
* Represents the current status of the system.
@@ -72,13 +63,14 @@ class SystemStatus {
*/
constructor(options = {}) {
const {
- currentHistorySecs,
- maxMemoryOverloadedRatio,
- maxEventLoopOverloadedRatio,
- maxCpuOverloadedRatio,
- maxClientOverloadedRatio,
+ currentHistorySecs = 5,
+ maxMemoryOverloadedRatio = 0.2,
+ maxEventLoopOverloadedRatio = 0.4,
+ maxCpuOverloadedRatio = 0.4,
+ maxClientOverloadedRatio = 0.3,
snapshotter,
- } = _.defaults({}, options, DEFAULT_OPTIONS);
+ } = options;
+
checkParamOrThrow(currentHistorySecs, 'options.currentHistorySecs', 'Number');
checkParamOrThrow(maxMemoryOverloadedRatio, 'options.maxMemoryOverloadedRatio', 'Number');
diff --git a/src/crawlers/basic_crawler.js b/src/crawlers/basic_crawler.js
index ab617d04f..706451c2c 100644
--- a/src/crawlers/basic_crawler.js
+++ b/src/crawlers/basic_crawler.js
@@ -6,10 +6,10 @@ import AutoscaledPool from '../autoscaling/autoscaled_pool'; // eslint-disable-l
import { RequestList } from '../request_list';
import { RequestQueue, RequestQueueLocal } from '../request_queue'; // eslint-disable-line import/no-duplicates
import events from '../events';
-import log from '../utils_log';
import { openSessionPool } from '../session_pool/session_pool'; // eslint-disable-line import/no-duplicates
import Statistics from './statistics';
import { addTimeoutToPromise } from '../utils';
+import defaultLog from '../utils_log';
// TYPE IMPORTS
/* eslint-disable no-unused-vars,import/named,import/no-duplicates,import/order */
@@ -33,18 +33,6 @@ import { SessionPoolOptions } from '../session_pool/session_pool';
*/
const SAFE_MIGRATION_WAIT_MILLIS = 20000;
-const DEFAULT_OPTIONS = {
- maxRequestRetries: 3,
- handleRequestTimeoutSecs: 60,
- handleFailedRequestFunction: ({ request }) => {
- const details = _.pick(request, 'id', 'url', 'method', 'uniqueKey');
- log.error('BasicCrawler: Request failed and reached maximum retries', details);
- },
- autoscaledPoolOptions: {},
- sessionPoolOptions: {}, // We could add sessionPool true/false config to use/not use SessionPool.
- useSessionPool: false,
-};
-
/**
* @typedef BasicCrawlerOptions
* @property {HandleRequest} handleRequestFunction
@@ -191,18 +179,24 @@ class BasicCrawler {
requestList,
requestQueue,
handleRequestFunction,
- handleRequestTimeoutSecs,
- handleFailedRequestFunction,
- maxRequestRetries,
+ handleRequestTimeoutSecs = 60,
+ handleFailedRequestFunction = ({ request }) => {
+ const details = _.pick(request, 'id', 'url', 'method', 'uniqueKey');
+ log.error('Request failed and reached maximum retries', details);
+ },
+ maxRequestRetries = 3,
maxRequestsPerCrawl,
- autoscaledPoolOptions,
- sessionPoolOptions,
- useSessionPool,
+ autoscaledPoolOptions = {},
+ sessionPoolOptions = {},
+ useSessionPool = false,
// AutoscaledPool shorthands
minConcurrency,
maxConcurrency,
- } = _.defaults({}, options, DEFAULT_OPTIONS);
+
+ // internal
+ log = defaultLog.child({ prefix: 'BasicCrawler' }),
+ } = options;
checkParamPrototypeOrThrow(requestList, 'options.requestList', RequestList, 'Apify.RequestList', true);
checkParamPrototypeOrThrow(requestQueue, 'options.requestQueue', [RequestQueue, RequestQueueLocal], 'Apify.RequestQueue', true);
@@ -219,6 +213,7 @@ class BasicCrawler {
throw new Error('At least one of the parameters "options.requestList" and "options.requestQueue" must be provided!');
}
+ this.log = log;
this.requestList = requestList;
this.requestQueue = requestQueue;
this.handleRequestFunction = handleRequestFunction;
@@ -226,8 +221,11 @@ class BasicCrawler {
this.handleFailedRequestFunction = handleFailedRequestFunction;
this.maxRequestRetries = maxRequestRetries;
this.handledRequestsCount = 0;
- this.stats = new Statistics({ logMessage: 'Crawler request statistics:' });
- this.sessionPoolOptions = sessionPoolOptions;
+ this.stats = new Statistics({ logMessage: `${log.getOptions().prefix} request statistics:` });
+ this.sessionPoolOptions = {
+ ...sessionPoolOptions,
+ log,
+ };
this.useSessionPool = useSessionPool;
let shouldLogMaxPagesExceeded = true;
@@ -242,7 +240,7 @@ class BasicCrawler {
isTaskReadyFunction: async () => {
if (isMaxPagesExceeded()) {
if (shouldLogMaxPagesExceeded) {
- log.info('BasicCrawler: Crawler reached the maxRequestsPerCrawl limit of '
+ log.info('Crawler reached the maxRequestsPerCrawl limit of '
+ `${maxRequestsPerCrawl} requests and will shut down soon. Requests that are in progress will be allowed to finish.`);
shouldLogMaxPagesExceeded = false;
}
@@ -253,7 +251,7 @@ class BasicCrawler {
},
isFinishedFunction: async () => {
if (isMaxPagesExceeded()) {
- log.info(`BasicCrawler: Earlier, the crawler reached the maxRequestsPerCrawl limit of ${maxRequestsPerCrawl} requests `
+ log.info(`Earlier, the crawler reached the maxRequestsPerCrawl limit of ${maxRequestsPerCrawl} requests `
+ 'and all requests that were in progress at that time have now finished. '
+ `In total, the crawler processed ${this.handledRequestsCount} requests and will shut down.`);
return true;
@@ -265,13 +263,14 @@ class BasicCrawler {
if (isFinished) {
const reason = isFinishedFunction
- ? 'BasicCrawler: Crawler\'s custom isFinishedFunction() returned true, the crawler will shut down.'
- : 'BasicCrawler: All the requests from request list and/or request queue have been processed, the crawler will shut down.';
+ ? 'Crawler\'s custom isFinishedFunction() returned true, the crawler will shut down.'
+ : 'All the requests from request list and/or request queue have been processed, the crawler will shut down.';
log.info(reason);
}
return isFinished;
},
+ log,
};
this.autoscaledPoolOptions = _.defaults({}, basicCrawlerAutoscaledPoolConfiguration, autoscaledPoolOptions);
@@ -312,7 +311,7 @@ class BasicCrawler {
this.stats.stopLogging();
const finalStats = this.stats.getCurrent();
- log.info('Crawler final request statistics:', finalStats);
+ this.log.info('Final request statistics:', finalStats);
}
}
@@ -320,7 +319,7 @@ class BasicCrawler {
await this.autoscaledPool.pause(SAFE_MIGRATION_WAIT_MILLIS)
.catch((err) => {
if (err.message.includes('running tasks did not finish')) {
- log.error('BasicCrawler: The crawler was paused due to migration to another host, '
+ this.log.error('The crawler was paused due to migration to another host, '
+ 'but some requests did not finish in time. Those requests\' results may be duplicated.');
} else {
throw err;
@@ -331,11 +330,11 @@ class BasicCrawler {
await this.requestList.persistState()
.catch((err) => {
if (err.message.includes('Cannot persist state.')) {
- log.error('BasicCrawler: The crawler attempted to persist its request list\'s state and failed due to missing or '
+ this.log.error('The crawler attempted to persist its request list\'s state and failed due to missing or '
+ 'invalid config. Make sure to use either Apify.openRequestList() or the "stateKeyPrefix" option of RequestList '
+ 'constructor to ensure your crawling state is persisted through host migrations and restarts.');
} else {
- log.exception(err, 'BasicCrawler: An unexpected error occured when the crawler '
+ this.log.exception(err, 'An unexpected error occured when the crawler '
+ 'attempted to persist its request list\'s state.');
}
});
@@ -359,7 +358,7 @@ class BasicCrawler {
} catch (err) {
// If requestQueue.addRequest() fails here then we must reclaim it back to
// the RequestList because probably it's not yet in the queue!
- log.exception(err, 'RequestQueue.addRequest() failed, reclaiming request back to the list', { request });
+ this.log.exception(err, 'RequestQueue.addRequest() failed, reclaiming request back to the list', { request });
await this.requestList.reclaimRequest(request);
return null;
}
@@ -399,7 +398,7 @@ class BasicCrawler {
await addTimeoutToPromise(
this.handleRequestFunction({ request, autoscaledPool: this.autoscaledPool, session }),
this.handleRequestTimeoutMillis,
- `BasicCrawler: handleRequestFunction timed out after ${this.handleRequestTimeoutMillis / 1000} seconds.`,
+ `handleRequestFunction timed out after ${this.handleRequestTimeoutMillis / 1000} seconds.`,
);
await source.markRequestHandled(request);
this.stats.finishJob(statisticsId);
@@ -411,7 +410,7 @@ class BasicCrawler {
try {
await this._requestFunctionErrorHandler(err, request, source);
} catch (secondaryError) {
- log.exception(secondaryError, 'BasicCrawler: runTaskFunction error handler threw an exception. '
+ this.log.exception(secondaryError, 'runTaskFunction error handler threw an exception. '
+ 'This places the crawler and its underlying storages into an unknown state and crawling will be terminated. '
+ 'This may have happened due to an internal error of Apify\'s API or due to a misconfigured crawler. '
+ 'If you are sure that there is no error in your code, selecting "Restart on error" in the actor\'s settings'
@@ -466,9 +465,9 @@ class BasicCrawler {
// Reclaim and retry request if flagged as retriable and retryCount is not exceeded.
if (!request.noRetry && request.retryCount < this.maxRequestRetries) {
request.retryCount++;
- log.exception(
+ this.log.exception(
error,
- 'BasicCrawler: handleRequestFunction failed, reclaiming failed request back to the list or queue',
+ 'handleRequestFunction failed, reclaiming failed request back to the list or queue',
_.pick(request, 'url', 'retryCount', 'id'),
);
return source.reclaimRequest(request);
diff --git a/src/crawlers/cheerio_crawler.js b/src/crawlers/cheerio_crawler.js
index c5d02b55c..e3e022bdd 100644
--- a/src/crawlers/cheerio_crawler.js
+++ b/src/crawlers/cheerio_crawler.js
@@ -13,7 +13,7 @@ import { TimeoutError } from '../errors';
import { addTimeoutToPromise, parseContentTypeFromResponse } from '../utils';
import * as utilsRequest from '../utils_request'; // eslint-disable-line import/no-duplicates
import BasicCrawler from './basic_crawler'; // eslint-disable-line import/no-duplicates
-import log from '../utils_log';
+import defaultLog from '../utils_log';
// TYPE IMPORTS
/* eslint-disable no-unused-vars,import/named,import/no-duplicates,import/order */
@@ -330,7 +330,7 @@ class CheerioCrawler {
requestQueue,
maxRequestRetries,
maxRequestsPerCrawl,
- handleFailedRequestFunction = this._defaultHandleFailedRequestFunction,
+ handleFailedRequestFunction = this._defaultHandleFailedRequestFunction.bind(this),
autoscaledPoolOptions = DEFAULT_AUTOSCALED_POOL_OPTIONS,
prepareRequestFunction,
useSessionPool = false,
@@ -355,6 +355,8 @@ class CheerioCrawler {
checkParamOrThrow(sessionPoolOptions, 'options.sessionPoolOptions', 'Object');
checkParamOrThrow(persistCookiesPerSession, 'options.persistCookiesPerSession', 'Boolean');
+ this.log = defaultLog.child({ prefix: 'CheerioCrawler' });
+
// Enforce valid proxy configuration
if (proxyUrls && !proxyUrls.length) throw new Error('Parameter "options.proxyUrls" of type Array must not be empty');
if (useApifyProxy && proxyUrls) throw new Error('Cannot combine "options.useApifyProxy" with "options.proxyUrls"!');
@@ -373,11 +375,11 @@ class CheerioCrawler {
if (requestOptions) {
// DEPRECATED 2020-03-22
this.requestOptions = requestOptions;
- log.deprecated('CheerioCrawler: options.requestOptions is deprecated. Use options.prepareRequestFunction instead.');
+ this.log.deprecated('options.requestOptions is deprecated. Use options.prepareRequestFunction instead.');
}
if (suggestResponseEncoding && forceResponseEncoding) {
- log.warning('CheerioCrawler: Both forceResponseEncoding and suggestResponseEncoding options are set. Using forceResponseEncoding.');
+ this.log.warning('Both forceResponseEncoding and suggestResponseEncoding options are set. Using forceResponseEncoding.');
}
this.handlePageFunction = handlePageFunction;
@@ -414,6 +416,9 @@ class CheerioCrawler {
// Session pool options
sessionPoolOptions,
useSessionPool,
+
+ // log
+ log: this.log,
});
this.isRunningPromise = null;
@@ -447,7 +452,7 @@ class CheerioCrawler {
const { dom, isXml, body, contentType, response } = await addTimeoutToPromise(
this._requestFunction({ request, session }),
this.requestTimeoutMillis,
- `CheerioCrawler: request timed out after ${this.requestTimeoutMillis / 1000} seconds.`,
+ `request timed out after ${this.requestTimeoutMillis / 1000} seconds.`,
);
if (this.useSessionPool) {
@@ -459,6 +464,7 @@ class CheerioCrawler {
}
request.loadedUrl = response.url;
+ const { log } = this;
const $ = dom ? cheerio.load(dom, { xmlMode: isXml }) : null;
const context = {
@@ -466,7 +472,7 @@ class CheerioCrawler {
// Using a getter here not to break the original API
// and lazy load the HTML only when needed.
get html() {
- log.deprecated('CheerioCrawler: The "html" parameter of handlePageFunction is deprecated, use "body" instead.');
+ log.deprecated('The "html" parameter of handlePageFunction is deprecated, use "body" instead.');
return dom && !isXml && $.html({ decodeEntities: false });
},
get json() {
@@ -492,7 +498,7 @@ class CheerioCrawler {
return addTimeoutToPromise(
this.handlePageFunction(context),
this.handlePageTimeoutMillis,
- `CheerioCrawler: handlePageFunction timed out after ${this.handlePageTimeoutMillis / 1000} seconds.`,
+ `handlePageFunction timed out after ${this.handlePageTimeoutMillis / 1000} seconds.`,
);
}
@@ -546,7 +552,7 @@ class CheerioCrawler {
}
// It's not a JSON so it's probably some text. Get the first 100 chars of it.
- throw new Error(`CheerioCrawler: ${statusCode} - Internal Server Error: ${body.substr(0, 100)}`);
+ throw new Error(`${statusCode} - Internal Server Error: ${body.substr(0, 100)}`);
} else if (type === 'text/html' || type === 'application/xhtml+xml' || type === 'application/xml') {
const dom = await this._parseHtmlToDom(response);
return ({ dom, isXml: type.includes('xml'), response, contentType });
@@ -577,12 +583,12 @@ class CheerioCrawler {
if (statusCode === 406) {
request.noRetry = true;
- throw new Error(`CheerioCrawler: Resource ${request.url} is not available in HTML format. Skipping resource.`);
+ throw new Error(`Resource ${request.url} is not available in HTML format. Skipping resource.`);
}
if (!this.supportedMimeTypes.has(type) && statusCode < 500) {
request.noRetry = true;
- throw new Error(`CheerioCrawler: Resource ${request.url} served Content-Type ${type}, `
+ throw new Error(`Resource ${request.url} served Content-Type ${type}, `
+ `but only ${Array.from(this.supportedMimeTypes).join(', ')} are allowed. Skipping resource.`);
}
@@ -648,7 +654,7 @@ class CheerioCrawler {
};
}
- throw new Error(`CheerioCrawler: Resource ${request.url} served with unsupported charset/encoding: ${encoding}`);
+ throw new Error(`Resource ${request.url} served with unsupported charset/encoding: ${encoding}`);
}
async _parseHtmlToDom(response) {
@@ -673,7 +679,7 @@ class CheerioCrawler {
const parsedType = contentTypeParser.parse(mimeType);
this.supportedMimeTypes.add(parsedType.type);
} catch (err) {
- throw new Error(`CheerioCrawler: Can not parse mime type ${mimeType} from "options.additionalMimeTypes".`);
+ throw new Error(`Can not parse mime type ${mimeType} from "options.additionalMimeTypes".`);
}
});
}
@@ -688,7 +694,7 @@ class CheerioCrawler {
const isBlocked = session.retireOnBlockedStatusCodes(statusCode);
if (isBlocked) {
- throw new Error(`CheerioCrawler: Request blocked - received ${statusCode} status code`);
+ throw new Error(`Request blocked - received ${statusCode} status code`);
}
}
@@ -699,7 +705,7 @@ class CheerioCrawler {
*/
_handleRequestTimeout(session) {
if (session) session.markBad();
- throw new Error(`CheerioCrawler: request timed out after ${this.handlePageTimeoutMillis / 1000} seconds.`);
+ throw new Error(`request timed out after ${this.handlePageTimeoutMillis / 1000} seconds.`);
}
/**
@@ -711,7 +717,7 @@ class CheerioCrawler {
*/
async _defaultHandleFailedRequestFunction({ error, request }) { // eslint-disable-line class-methods-use-this
const details = _.pick(request, 'id', 'url', 'method', 'uniqueKey');
- log.exception(error, 'CheerioCrawler: Request failed and reached maximum retries', details);
+ this.log.exception(error, 'Request failed and reached maximum retries', details);
}
}
diff --git a/src/crawlers/puppeteer_crawler.js b/src/crawlers/puppeteer_crawler.js
index 84f0e5176..4e567b403 100644
--- a/src/crawlers/puppeteer_crawler.js
+++ b/src/crawlers/puppeteer_crawler.js
@@ -6,7 +6,7 @@ import { gotoExtended } from '../puppeteer_utils';
import { openSessionPool } from '../session_pool/session_pool'; // eslint-disable-line import/no-duplicates
import { addTimeoutToPromise } from '../utils';
import BasicCrawler from './basic_crawler'; // eslint-disable-line import/no-duplicates
-import log from '../utils_log';
+import defaultLog from '../utils_log';
// TYPE IMPORTS
/* eslint-disable no-unused-vars,import/named,import/no-duplicates,import/order */
@@ -228,7 +228,7 @@ class PuppeteerCrawler {
requestQueue,
maxRequestRetries,
maxRequestsPerCrawl,
- handleFailedRequestFunction = this._defaultHandleFailedRequestFunction,
+ handleFailedRequestFunction = this._defaultHandleFailedRequestFunction.bind(this),
autoscaledPoolOptions,
// PuppeteerPool options and shorthands
@@ -251,8 +251,10 @@ class PuppeteerCrawler {
checkParamOrThrow(sessionPoolOptions, 'options.sessionPoolOptions', 'Object');
checkParamOrThrow(persistCookiesPerSession, 'options.persistCookiesPerSession', 'Boolean');
+ this.log = defaultLog.child({ prefix: 'PuppeteerCrawler' });
+
if (options.gotoTimeoutSecs && options.gotoFunction) {
- log.warning('PuppeteerCrawler: You are using gotoTimeoutSecs with a custom gotoFunction. '
+ this.log.warning('You are using gotoTimeoutSecs with a custom gotoFunction. '
+ 'The timeout value will not be used. With a custom gotoFunction, you need to set the timeout in the function itself.');
}
@@ -270,7 +272,10 @@ class PuppeteerCrawler {
this.puppeteerPool = null; // Constructed when .run()
this.useSessionPool = useSessionPool;
- this.sessionPoolOptions = sessionPoolOptions;
+ this.sessionPoolOptions = {
+ ...sessionPoolOptions,
+ log: this.log,
+ };
this.persistCookiesPerSession = persistCookiesPerSession;
/** @ignore */
@@ -288,6 +293,9 @@ class PuppeteerCrawler {
maxConcurrency,
minConcurrency,
autoscaledPoolOptions,
+
+ // log
+ log: this.log,
});
}
@@ -303,6 +311,7 @@ class PuppeteerCrawler {
this.sessionPool = await openSessionPool(this.sessionPoolOptions);
this.puppeteerPoolOptions.sessionPool = this.sessionPool;
}
+ this.puppeteerPoolOptions.log = this.log;
this.puppeteerPool = new PuppeteerPool(this.puppeteerPoolOptions);
try {
this.isRunningPromise = this.basicCrawler.run();
@@ -353,7 +362,7 @@ class PuppeteerCrawler {
await addTimeoutToPromise(
this.handlePageFunction({ page, request, autoscaledPool, puppeteerPool: this.puppeteerPool, response, session }),
this.handlePageTimeoutMillis,
- `PuppeteerCrawler: handlePageFunction timed out after ${this.handlePageTimeoutMillis / 1000} seconds.`,
+ `handlePageFunction timed out after ${this.handlePageTimeoutMillis / 1000} seconds.`,
);
if (session) session.markGood();
@@ -382,7 +391,7 @@ class PuppeteerCrawler {
*/
async _defaultHandleFailedRequestFunction({ error, request }) { // eslint-disable-line class-methods-use-this
const details = _.pick(request, 'id', 'url', 'method', 'uniqueKey');
- log.exception(error, 'PuppeteerCrawler: Request failed and reached maximum retries', details);
+ this.log.exception(error, 'Request failed and reached maximum retries', details);
}
}
diff --git a/src/crawlers/statistics.js b/src/crawlers/statistics.js
index 231540731..14bd3e8e5 100644
--- a/src/crawlers/statistics.js
+++ b/src/crawlers/statistics.js
@@ -44,6 +44,7 @@ export default class Statistics {
checkParamOrThrow(logIntervalSecs, 'options.logIntervalSecs', 'Number');
checkParamOrThrow(logMessage, 'options.logMessage', 'String');
+
this.logIntervalMillis = logIntervalSecs * 1000;
this.logMessage = logMessage;
diff --git a/src/dataset.js b/src/dataset.js
index 53340b0d8..30540f53c 100644
--- a/src/dataset.js
+++ b/src/dataset.js
@@ -6,8 +6,8 @@ import { leftpad } from 'apify-shared/utilities';
import * as LruCache from 'apify-shared/lru_cache';
import { checkParamOrThrow } from 'apify-client/build/utils';
import { ENV_VARS, LOCAL_STORAGE_SUBDIRS, MAX_PAYLOAD_SIZE_BYTES } from 'apify-shared/consts';
-import log from './utils_log';
import { apifyClient, ensureDirExists, openRemoteStorage, openLocalStorage, ensureTokenOrLocalStorageEnvExists } from './utils';
+import log from './utils_log';
export const DATASET_ITERATORS_DEFAULT_LIMIT = 10000;
export const LOCAL_STORAGE_SUBDIR = LOCAL_STORAGE_SUBDIRS.datasets;
@@ -155,6 +155,7 @@ export class Dataset {
this.datasetId = datasetId;
this.datasetName = datasetName;
+ this.log = log.child({ prefix: 'Dataset' });
}
/**
@@ -274,9 +275,7 @@ export class Dataset {
return await datasets.getItems(params);
} catch (e) {
if (e.message.includes('Cannot create a string longer than')) {
- throw new Error(
- 'dataset.getData(): The response is too large for parsing. You can fix this by lowering the "limit" option.',
- );
+ throw new Error('dataset.getData(): The response is too large for parsing. You can fix this by lowering the "limit" option.');
}
throw e;
}
@@ -440,7 +439,7 @@ export class Dataset {
/** @ignore */
async delete() {
- log.deprecated('dataset.delete() is deprecated. Please use dataset.drop() instead. '
+ this.log.deprecated('dataset.delete() is deprecated. Please use dataset.drop() instead. '
+ 'This is to make it more obvious to users that the function deletes the dataset and not individual records in the dataset.');
await this.drop();
}
@@ -456,6 +455,7 @@ export class DatasetLocal {
checkParamOrThrow(datasetId, 'datasetId', 'String');
checkParamOrThrow(localStorageDir, 'localStorageDir', 'String');
+ this.log = log.child({ prefix: 'Dataset' });
this.localStoragePath = path.resolve(path.join(localStorageDir, LOCAL_STORAGE_SUBDIR, datasetId));
this.counter = null;
this.datasetId = datasetId;
@@ -603,7 +603,7 @@ export class DatasetLocal {
}
async delete() {
- log.deprecated('dataset.delete() is deprecated. Please use dataset.drop() instead. '
+ this.log.deprecated('dataset.delete() is deprecated. Please use dataset.drop() instead. '
+ 'This is to make it more obvious to users that the function deletes the dataset and not individual records in the dataset.');
await this.drop();
}
diff --git a/src/events.js b/src/events.js
index 071961582..eecdaed6b 100644
--- a/src/events.js
+++ b/src/events.js
@@ -1,8 +1,8 @@
import { EventEmitter } from 'events';
import * as WebSocket from 'ws';
import { ENV_VARS, ACTOR_EVENT_NAMES } from 'apify-shared/consts';
-import log from './utils_log';
import { ACTOR_EVENT_NAMES_EX } from './constants';
+import defaultLog from './utils_log';
// NOTE: This value is mentioned below in docs, if you update it here, update it there too.
const PERSIST_STATE_INTERVAL_MILLIS = 60 * 1000;
@@ -115,6 +115,8 @@ const emitPersistStateEvent = (isMigrating = false) => {
export const initializeEvents = () => {
if (eventsWs) return;
+ const log = defaultLog.child({ prefix: 'Events' });
+
if (!persistStateInterval) {
// This is overridable only to enable unit testing.
const intervalMillis = process.env.APIFY_TEST_PERSIST_INTERVAL_MILLIS || PERSIST_STATE_INTERVAL_MILLIS;
@@ -125,7 +127,7 @@ export const initializeEvents = () => {
// Locally there is no web socket to connect, so just print a log message.
if (!eventsWsUrl) {
- log.debug(`Apify.events: Environment variable ${ENV_VARS.ACTOR_EVENTS_WS_URL} is not set, no events from Apify platform will be emitted.`);
+ log.debug(`Environment variable ${ENV_VARS.ACTOR_EVENTS_WS_URL} is not set, no events from Apify platform will be emitted.`);
return;
}
@@ -143,17 +145,17 @@ export const initializeEvents = () => {
emitPersistStateEvent(true);
}
} catch (err) {
- log.exception(err, 'Apify.events: Cannot parse actor event');
+ log.exception(err, 'Cannot parse actor event');
}
});
eventsWs.on('error', (err) => {
// Don't print this error as this happens in the case of very short Apify.main().
if (err.message === 'WebSocket was closed before the connection was established') return;
- log.exception(err, 'Apify.events: web socket connection failed');
+ log.exception(err, 'web socket connection failed');
});
eventsWs.on('close', () => {
- log.warning('Apify.events: web socket has been closed');
+ log.warning('web socket has been closed');
eventsWs = null;
});
};
diff --git a/src/key_value_store.js b/src/key_value_store.js
index d199dc487..3dd809c96 100644
--- a/src/key_value_store.js
+++ b/src/key_value_store.js
@@ -8,11 +8,11 @@ import { KEY_VALUE_STORE_KEY_REGEX } from 'apify-shared/regexs';
import { ENV_VARS, LOCAL_STORAGE_SUBDIRS, KEY_VALUE_STORE_KEYS } from 'apify-shared/consts';
import { jsonStringifyExtended } from 'apify-shared/utilities';
import { checkParamOrThrow, parseBody } from 'apify-client/build/utils';
-import log from './utils_log';
import {
addCharsetToContentType, apifyClient, ensureDirExists, openRemoteStorage, openLocalStorage, ensureTokenOrLocalStorageEnvExists,
} from './utils';
import { APIFY_API_BASE_URL } from './constants';
+import log from './utils_log';
export const LOCAL_STORAGE_SUBDIR = LOCAL_STORAGE_SUBDIRS.keyValueStores;
const MAX_OPENED_STORES = 1000;
@@ -171,6 +171,7 @@ export class KeyValueStore {
this.storeId = storeId;
this.storeName = storeName;
+ this.log = log.child({ prefix: 'KeyValueStore' });
}
/**
@@ -298,7 +299,7 @@ export class KeyValueStore {
/** @ignore */
async delete() {
- log.deprecated('keyValueStore.delete() is deprecated. Please use keyValueStore.drop() instead. '
+ this.log.deprecated('keyValueStore.delete() is deprecated. Please use keyValueStore.drop() instead. '
+ 'This is to make it more obvious to users that the function deletes the key-value store and not individual records in the store.');
await this.drop();
}
@@ -376,6 +377,7 @@ export class KeyValueStoreLocal {
checkParamOrThrow(storeId, 'storeId', 'String');
checkParamOrThrow(localStorageDir, 'localStorageDir', 'String');
+ this.log = log.child({ prefix: 'KeyValueStore' });
this.localStoragePath = path.resolve(path.join(localStorageDir, LOCAL_STORAGE_SUBDIR, storeId));
this.storeId = storeId;
this.initializationPromise = ensureDirExists(this.localStoragePath);
@@ -428,7 +430,7 @@ export class KeyValueStoreLocal {
}
async delete() {
- log.deprecated('keyValueStore.delete() is deprecated. Please use keyValueStore.drop() instead. '
+ this.log.deprecated('keyValueStore.delete() is deprecated. Please use keyValueStore.drop() instead. '
+ 'This is to make it more obvious to users that the function deletes the key-value store and not individual records in the store.');
await this.drop();
}
diff --git a/src/live_view/live_view_server.js b/src/live_view/live_view_server.js
index 28799142f..f2a1974ca 100644
--- a/src/live_view/live_view_server.js
+++ b/src/live_view/live_view_server.js
@@ -8,9 +8,9 @@ import { checkParamOrThrow } from 'apify-client/build/utils';
import { promisifyServerListen } from 'apify-shared/utilities';
import { ENV_VARS, LOCAL_ENV_VARS } from 'apify-shared/consts';
import { Page } from 'puppeteer'; // eslint-disable-line no-unused-vars
-import log from '../utils_log';
import { addTimeoutToPromise } from '../utils';
import Snapshot from './snapshot';
+import defaultLog from '../utils_log';
const writeFile = promisify(fs.writeFile);
const unlink = promisify(fs.unlink);
@@ -84,6 +84,8 @@ class LiveViewServer {
checkParamOrThrow(maxScreenshotFiles, 'options.maxScreenshotFiles', 'Number');
checkParamOrThrow(maxScreenshotFiles, 'options.snapshotTimeoutSecs', 'Number');
checkParamOrThrow(maxScreenshotFiles, 'options.maxSnapshotFrequencySecs', 'Number');
+
+ this.log = defaultLog.child({ prefix: 'LiveViewServer' });
this.screenshotDirectoryPath = screenshotDirectoryPath;
this.maxScreenshotFiles = maxScreenshotFiles;
this.snapshotTimeoutMillis = snapshotTimeoutSecs * 1000;
@@ -116,9 +118,9 @@ class LiveViewServer {
try {
await ensureDir(this.screenshotDirectoryPath);
await promisifyServerListen(this.httpServer)(this.port);
- log.info('Live view web server started', { publicUrl: this.liveViewUrl });
+ this.log.info('Live view web server started', { publicUrl: this.liveViewUrl });
} catch (err) {
- log.exception(err, 'Live view web server failed to start.');
+ this.log.exception(err, 'Live view web server failed to start.');
this._isRunning = false;
}
}
@@ -133,8 +135,8 @@ class LiveViewServer {
return new Promise((resolve) => {
this.httpServer.close((err) => {
this._isRunning = false;
- if (err) log.exception(err, 'Live view web server could not be stopped.');
- else log.info('Live view web server stopped.');
+ if (err) this.log.exception(err, 'Live view web server could not be stopped.');
+ else this.log.info('Live view web server stopped.');
resolve();
});
});
@@ -205,7 +207,7 @@ class LiveViewServer {
*/
async _makeSnapshot(page) {
const pageUrl = page.url();
- log.info('Making live view snapshot.', { pageUrl });
+ this.log.info('Making live view snapshot.', { pageUrl });
const [htmlContent, screenshot] = await Promise.all([
page.content(),
page.screenshot({
@@ -232,7 +234,7 @@ class LiveViewServer {
*/
_pushSnapshot(snapshot) {
// Send new snapshot to clients
- log.debug('Sending live view snapshot', { createdAt: snapshot.createdAt, pageUrl: snapshot.pageUrl });
+ this.log.debug('Sending live view snapshot', { createdAt: snapshot.createdAt, pageUrl: snapshot.pageUrl });
this.socketio.emit('snapshot', snapshot);
}
@@ -243,7 +245,7 @@ class LiveViewServer {
*/
_deleteScreenshot(screenshotIndex) {
unlink(this._getScreenshotPath(screenshotIndex))
- .catch(err => log.exception(err, 'Cannot delete live view screenshot.'));
+ .catch(err => this.log.exception(err, 'Cannot delete live view screenshot.'));
}
_setupHttpServer() {
@@ -285,14 +287,14 @@ class LiveViewServer {
*/
_socketConnectionHandler(socket) {
this.clientCount++;
- log.info('Live view client connected', { clientId: socket.id });
+ this.log.info('Live view client connected', { clientId: socket.id });
socket.on('disconnect', (reason) => {
this.clientCount--;
- log.info('Live view client disconnected', { clientId: socket.id, reason });
+ this.log.info('Live view client disconnected', { clientId: socket.id, reason });
});
socket.on('getLastSnapshot', () => {
if (this.lastSnapshot) {
- log.debug('Sending live view snapshot', { createdAt: this.lastSnapshot.createdAt, pageUrl: this.lastSnapshot.pageUrl });
+ this.log.debug('Sending live view snapshot', { createdAt: this.lastSnapshot.createdAt, pageUrl: this.lastSnapshot.pageUrl });
this.socketio.emit('snapshot', this.lastSnapshot);
}
});
diff --git a/src/puppeteer_pool.js b/src/puppeteer_pool.js
index 0f93b5d13..dbd35af94 100644
--- a/src/puppeteer_pool.js
+++ b/src/puppeteer_pool.js
@@ -6,8 +6,7 @@ import * as util from 'util';
import * as LinkedList from 'apify-shared/linked_list';
import * as rimraf from 'rimraf';
import { checkParamOrThrow } from 'apify-client/build/utils';
-import log from './utils_log';
-
+import defaultLog from './utils_log';
import { addTimeoutToPromise } from './utils';
import LiveViewServer from './live_view/live_view_server';
import EVENTS from './session_pool/events';
@@ -24,41 +23,10 @@ export const BROWSER_SESSION_KEY_NAME = 'APIFY_SESSION';
const PROCESS_KILL_TIMEOUT_MILLIS = 5000;
const PAGE_CLOSE_KILL_TIMEOUT_MILLIS = 1000;
-const DEFAULT_OPTIONS = {
- reusePages: false,
- // Don't make these too large, otherwise Puppeteer might start crashing weirdly,
- // and the default settings should just work
- maxOpenPagesPerInstance: 50,
- retireInstanceAfterRequestCount: 100,
-
- puppeteerOperationTimeoutSecs: 15,
- instanceKillerIntervalSecs: 60,
- killInstanceAfterSecs: 300,
-
- launchPuppeteerFunction: launchPuppeteerOptions => launchPuppeteer(launchPuppeteerOptions),
-
- recycleDiskCache: false,
- useIncognitoPages: false,
- useLiveView: false,
-};
-
const mkdtempAsync = util.promisify(fs.mkdtemp);
const rimrafAsync = util.promisify(rimraf);
const DISK_CACHE_DIR = path.join(os.tmpdir(), 'puppeteer_disk_cache-');
-/**
- * Deletes Chrome's user data directory
- * @param {string} diskCacheDir
- * @ignore
- */
-const deleteDiskCacheDir = (diskCacheDir) => {
- log.debug('PuppeteerPool: Deleting disk cache directory', { diskCacheDir });
- return rimrafAsync(diskCacheDir)
- .catch((err) => {
- log.warning('PuppeteerPool: Cannot delete Chrome disk cache directory', { diskCacheDir, errorMessage: err.message });
- });
-};
-
/**
* Internal representation of Puppeteer instance.
*
@@ -182,21 +150,25 @@ class PuppeteerPool {
*/
constructor(options = {}) {
const {
- maxOpenPagesPerInstance,
- retireInstanceAfterRequestCount,
- launchPuppeteerFunction,
- puppeteerOperationTimeoutSecs,
+ // Don't make these too large, otherwise Puppeteer might start crashing weirdly,
+ // and the default settings should just work
+ maxOpenPagesPerInstance = 50,
+ retireInstanceAfterRequestCount = 100,
+ launchPuppeteerFunction = launchPuppeteerOptions => launchPuppeteer(launchPuppeteerOptions),
+ puppeteerOperationTimeoutSecs = 15,
instanceKillerIntervalMillis,
- instanceKillerIntervalSecs,
+ instanceKillerIntervalSecs = 60,
killInstanceAfterMillis,
- killInstanceAfterSecs,
+ killInstanceAfterSecs = 300,
launchPuppeteerOptions,
- recycleDiskCache,
- useIncognitoPages,
+ recycleDiskCache = false,
+ useIncognitoPages = false,
proxyUrls,
- useLiveView,
+ useLiveView = false,
sessionPool,
- } = _.defaults({}, options, DEFAULT_OPTIONS);
+ log = defaultLog,
+ } = options;
+
// Disabling due to memory leak.
const reusePages = false;
@@ -207,13 +179,16 @@ class PuppeteerPool {
checkParamOrThrow(launchPuppeteerFunction, 'options.launchPuppeteerFunction', 'Function');
checkParamOrThrow(puppeteerOperationTimeoutSecs, 'options.puppeteerOperationTimeoutSecs', 'Number');
checkParamOrThrow(instanceKillerIntervalMillis, 'options.instanceKillerIntervalMillis', 'Maybe Number');
+
+ this.log = log.child({ prefix: 'PuppeteerPool' });
+
if (instanceKillerIntervalMillis) {
- log.deprecated('PuppeteerPool: options.instanceKillerIntervalMillis is deprecated, use options.instanceKillerIntervalSecs instead.');
+ this.log.deprecated('options.instanceKillerIntervalMillis is deprecated, use options.instanceKillerIntervalSecs instead.');
}
checkParamOrThrow(instanceKillerIntervalSecs, 'options.instanceKillerIntervalSecs', 'Number');
checkParamOrThrow(killInstanceAfterMillis, 'options.killInstanceAfterMillis', 'Maybe Number');
if (killInstanceAfterMillis) {
- log.deprecated('PuppeteerPool: options.killInstanceAfterMillis is deprecated, use options.killInstanceAfterSecs instead.');
+ this.log.deprecated('options.killInstanceAfterMillis is deprecated, use options.killInstanceAfterSecs instead.');
}
checkParamOrThrow(killInstanceAfterSecs, 'options.killInstanceAfterSecs', 'Number');
checkParamOrThrow(launchPuppeteerOptions, 'options.launchPuppeteerOptions', 'Maybe Object');
@@ -317,7 +292,7 @@ class PuppeteerPool {
*/
_launchInstance() {
const id = this.browserCounter++;
- log.debug('PuppeteerPool: Launching new browser', { id });
+ this.log.debug('Launching new browser', { id });
const browserPromise = this.launchPuppeteerFunction();
@@ -347,7 +322,7 @@ class PuppeteerPool {
instance.session = browser[BROWSER_SESSION_KEY_NAME];
}
} catch (err) {
- log.exception(err, 'PuppeteerPool: Browser launch failed', { id });
+ this.log.exception(err, 'Browser launch failed', { id });
delete this.activeInstances[id];
return;
}
@@ -357,7 +332,7 @@ class PuppeteerPool {
browser.on('disconnected', () => {
// If instance.killed === true then we killed the instance so don't log it.
- if (!instance.killed) log.error('PuppeteerPool: Puppeteer sent "disconnect" event. Maybe it crashed???', { id });
+ if (!instance.killed) this.log.error('Puppeteer sent "disconnect" event. Maybe it crashed???', { id });
this._retireInstance(instance);
});
}
@@ -370,9 +345,9 @@ class PuppeteerPool {
_retireInstance(instance) {
const { id } = instance;
- if (!this.activeInstances[id]) return log.debug('PuppeteerPool: browser is retired already', { id });
+ if (!this.activeInstances[id]) return this.log.debug('browser is retired already', { id });
- log.debug('PuppeteerPool: Retiring browser', { id });
+ this.log.debug('Retiring browser', { id });
this.retiredInstances[id] = instance;
delete this.activeInstances[id];
@@ -386,12 +361,12 @@ class PuppeteerPool {
* @ignore
*/
_killRetiredInstances() {
- log.debug('PuppeteerPool: Retired browsers count', { count: _.values(this.retiredInstances).length });
+ this.log.debug('Retired browsers count', { count: _.values(this.retiredInstances).length });
_.mapObject(this.retiredInstances, async (instance) => {
// Kill instances that are more than this.killInstanceAfterMillis from last opened page
if (Date.now() - instance.lastPageOpenedAt > this.killInstanceAfterMillis) {
- log.debug('PuppeteerPool: killing retired browser after period of inactivity', { id: instance.id, killInstanceAfterMillis: this.killInstanceAfterMillis }); // eslint-disable-line max-len
+ this.log.debug('killing retired browser after period of inactivity', { id: instance.id, killInstanceAfterMillis: this.killInstanceAfterMillis }); // eslint-disable-line max-len
this._killInstance(instance);
return;
}
@@ -401,11 +376,11 @@ class PuppeteerPool {
const pages = await browser.pages();
// NOTE: we are killing instance when the number of pages is less or equal to 1 because there is always about:blank page.
if (pages.length <= 1) {
- log.debug('PuppeteerPool: Killing retired browser because it has no open tabs', { id: instance.id });
+ this.log.debug('Killing retired browser because it has no open tabs', { id: instance.id });
this._killInstance(instance);
}
} catch (err) {
- log.exception(err, 'PuppeteerPool: Browser.pages() failed', { id: instance.id });
+ this.log.exception(err, 'Browser.pages() failed', { id: instance.id });
this._killInstance(instance);
}
});
@@ -420,13 +395,13 @@ class PuppeteerPool {
const { id, childProcess, killed, browserPromise } = instance;
if (killed) return;
- log.debug('PuppeteerPool: Killing browser', { id });
+ this.log.debug('Killing browser', { id });
delete this.retiredInstances[id];
const recycleDiskCache = () => {
if (!instance.recycleDiskCacheDir) return;
- log.debug('PuppeteerPool: Recycling disk cache dir', { id, diskCacheDir: instance.recycleDiskCacheDir });
+ this.log.debug('Recycling disk cache dir', { id, diskCacheDir: instance.recycleDiskCacheDir });
this.recycledDiskCacheDirs.add(instance.recycleDiskCacheDir);
instance.recycleDiskCacheDir = null;
};
@@ -447,7 +422,7 @@ class PuppeteerPool {
await browser.close();
recycleDiskCache();
} catch (err) {
- log.exception(err, 'PuppeteerPool: Cannot close the browser instance, it will be killed forcibly.', { id });
+ this.log.exception(err, 'Cannot close the browser instance, it will be killed forcibly.', { id });
}
}
@@ -539,14 +514,14 @@ class PuppeteerPool {
const page = await addTimeoutToPromise(
context.newPage(),
this.puppeteerOperationTimeoutMillis,
- 'PuppeteerPool: browser.newPage() timed out.',
+ 'browser.newPage() timed out.',
);
- await this._focusOldestTab(browser).catch(() => log.debug('Could not focus oldest tab.'));
+ await this._focusOldestTab(browser).catch(() => this.log.debug('Could not focus oldest tab.'));
this.pagesToInstancesMap.set(page, instance);
return this._decoratePage(page);
} catch (err) {
this._retireInstance(instance);
- const betterError = new Error(`PuppeteerPool: browser.newPage() failed: ${instance.id}.`);
+ const betterError = new Error(`browser.newPage() failed: ${instance.id}.`);
betterError.stack = err.stack;
throw betterError;
}
@@ -566,18 +541,18 @@ class PuppeteerPool {
this.closedPages.add(page);
await originalPageClose.apply(page, args)
.catch((err) => {
- log.debug('PuppeteerPool: Page.close() failed', { errorMessage: err.message, id: instance.id });
+ this.log.debug('Page.close() failed', { errorMessage: err.message, id: instance.id });
});
const context = page.browserContext();
if (context.isIncognito()) {
await context.close().catch((err) => {
- log.debug('PuppeteerPool: Context.close() failed', { errorMessage: err.message, id: instance.id });
+ this.log.debug('Context.close() failed', { errorMessage: err.message, id: instance.id });
});
}
};
page.once('error', (error) => {
- log.exception(error, 'PuppeteerPool: Page crashed.');
+ this.log.exception(error, 'Page crashed.');
page.close();
});
@@ -601,6 +576,20 @@ class PuppeteerPool {
if (pages.length > 1) return pages[1].bringToFront();
}
+ /**
+ * Deletes Chrome's user data directory
+ * @param {string} diskCacheDir
+ * @private
+ * @ignore
+ */
+ async _deleteDiskCacheDir(diskCacheDir) {
+ this.log.debug('Deleting disk cache directory', { diskCacheDir });
+ return rimrafAsync(diskCacheDir)
+ .catch((err) => {
+ this.log.warning('Cannot delete Chrome disk cache directory', { diskCacheDir, errorMessage: err.message });
+ });
+ }
+
/**
* Closes all open browsers.
* @return {Promise<void>}
@@ -624,7 +613,7 @@ class PuppeteerPool {
const closePromises = browserPromises.map(async (browserPromise) => {
const browser = await browserPromise;
await browser.close();
- if (browser.recycleDiskCacheDir) await deleteDiskCacheDir(browser.recycleDiskCacheDir);
+ if (browser.recycleDiskCacheDir) await this._deleteDiskCacheDir(browser.recycleDiskCacheDir);
});
try {
@@ -632,14 +621,14 @@ class PuppeteerPool {
// Delete all cache directories
const dirDeletionPromises = [];
while (this.recycledDiskCacheDirs && this.recycledDiskCacheDirs.length > 0) {
- dirDeletionPromises.push(deleteDiskCacheDir(this.recycledDiskCacheDirs.removeFirst()));
+ dirDeletionPromises.push(this._deleteDiskCacheDir(this.recycledDiskCacheDirs.removeFirst()));
}
await Promise.all(dirDeletionPromises);
} catch (err) {
- log.exception(err, 'PuppeteerPool: Cannot close the browsers.');
+ this.log.exception(err, 'Cannot close the browsers.');
}
if (this.liveViewServer) {
- await this.liveViewServer.stop().catch(err => log.exception(err, 'PuppeteerPool: Cannot close LiveViewServer.'));
+ await this.liveViewServer.stop().catch(err => this.log.exception(err, 'Cannot close LiveViewServer.'));
}
}
@@ -662,7 +651,7 @@ class PuppeteerPool {
case 1:
return results[0];
default:
- throw new Error('PuppeteerPool: Multiple instances of PuppeteerPool found using a single browser instance.');
+ throw new Error('Multiple instances of PuppeteerPool found using a single browser instance.');
}
}
@@ -677,7 +666,7 @@ class PuppeteerPool {
async retire(browser) {
const instance = await this._findInstanceByBrowser(browser);
if (instance) return this._retireInstance(instance);
- log.debug('PuppeteerPool: Browser is retired already');
+ this.log.debug('Browser is retired already');
}
/**
@@ -703,10 +692,10 @@ class PuppeteerPool {
await addTimeoutToPromise(
page.close(),
this.puppeteerOperationTimeoutMillis,
- 'PuppeteerPool: page.close() timed out.',
+ 'page.close() timed out.',
);
} catch (err) {
- log.debug('PuppeteerPool: page.close() failed.', { reason: err && err.message });
+ this.log.debug('page.close() failed.', { reason: err && err.message });
}
}
}
@@ -736,7 +725,7 @@ class PuppeteerPool {
if (instance.id !== this.browserCounter - 1) return;
const snapshotPromise = this.liveViewServer.serve(page)
- .catch(err => log.debug('Live View failed to be served.', { message: err.message }));
+ .catch(err => this.log.debug('Live View failed to be served.', { message: err.message }));
this.liveViewSnapshotsInProgress.set(page, snapshotPromise);
}
@@ -757,7 +746,7 @@ class PuppeteerPool {
} catch (e) {
// ignore the error, since the instance is either retired already, being retired or cannot be retired
// ( we cant do nothing about this at this point)
- log.debug('Could not retire instances ', e);
+ this.log.debug('Could not retire instances ', e);
}
}
}
@@ -768,7 +757,7 @@ class PuppeteerPool {
// Run this with a delay, otherwise page.close()
// might fail with "Protocol error (Target.closeTarget): Target closed."
setTimeout(() => {
- log.debug('PuppeteerPool: Killing retired browser because it has no active pages', { id });
+ this.log.debug('Killing retired browser because it has no active pages', { id });
this._killInstance(instance);
}, PAGE_CLOSE_KILL_TIMEOUT_MILLIS);
}
diff --git a/src/request.js b/src/request.js
index 7c87ecb47..9f2098d7e 100644
--- a/src/request.js
+++ b/src/request.js
@@ -3,23 +3,10 @@ import * as util from 'util';
import * as crypto from 'crypto';
import { checkParamOrThrow } from 'apify-client/build/utils';
import { normalizeUrl } from 'apify-shared/utilities';
-import log from './utils_log';
+import defaultLog from './utils_log';
-export function computeUniqueKey({ url, method, payload, keepUrlFragment, useExtendedUniqueKey }) {
- const normalizedMethod = method.toUpperCase();
- const normalizedUrl = normalizeUrl(url, keepUrlFragment) || url; // It returns null when url is invalid, causing weird errors.
- if (!useExtendedUniqueKey) {
- if (normalizedMethod !== 'GET' && payload) {
- // Using log.deprecated to log only once. We should add log.once or some such.
- log.deprecated(`We've encountered a ${normalizedMethod} Request with a payload. `
- + 'This is fine. Just letting you know that if your requests point to the same URL '
- + 'and differ only in method and payload, you should see the "useExtendedUniqueKey" option of Request constructor.');
- }
- return normalizedUrl;
- }
- const payloadHash = payload ? hashPayload(payload) : '';
- return `${normalizedMethod}(${payloadHash}):${normalizedUrl}`;
-}
+// new properties on the Request object breaks serialization
+const log = defaultLog.child({ prefix: 'Request' });
export function hashPayload(payload) {
return crypto
@@ -114,7 +101,6 @@ class Request {
useExtendedUniqueKey = false,
} = options;
-
checkParamOrThrow(id, 'id', 'Maybe String');
checkParamOrThrow(url, 'url', 'String');
checkParamOrThrow(loadedUrl, 'url', 'Maybe String');
@@ -137,7 +123,7 @@ class Request {
this.id = id;
this.url = url;
this.loadedUrl = loadedUrl;
- this.uniqueKey = uniqueKey || computeUniqueKey({ url, method, payload, keepUrlFragment, useExtendedUniqueKey });
+ this.uniqueKey = uniqueKey || this._computeUniqueKey({ url, method, payload, keepUrlFragment, useExtendedUniqueKey });
this.method = method;
this.payload = payload;
this.noRetry = noRetry;
@@ -218,6 +204,26 @@ class Request {
this.noRetry = true;
if (message) throw new Error(message);
}
+
+ /**
+ * @ignore
+ * @private
+ */
+ _computeUniqueKey({ url, method, payload, keepUrlFragment, useExtendedUniqueKey }) {
+ const normalizedMethod = method.toUpperCase();
+ const normalizedUrl = normalizeUrl(url, keepUrlFragment) || url; // It returns null when url is invalid, causing weird errors.
+ if (!useExtendedUniqueKey) {
+ if (normalizedMethod !== 'GET' && payload) {
+ // Using log.deprecated to log only once. We should add log.once or some such.
+ log.deprecated(`We've encountered a ${normalizedMethod} Request with a payload. `
+ + 'This is fine. Just letting you know that if your requests point to the same URL '
+ + 'and differ only in method and payload, you should see the "useExtendedUniqueKey" option of Request constructor.');
+ }
+ return normalizedUrl;
+ }
+ const payloadHash = payload ? hashPayload(payload) : '';
+ return `${normalizedMethod}(${payloadHash}):${normalizedUrl}`;
+ }
}
export default Request;
diff --git a/src/request_list.js b/src/request_list.js
index cc37f3f2f..243fe35a1 100644
--- a/src/request_list.js
+++ b/src/request_list.js
@@ -2,8 +2,8 @@ import { checkParamOrThrow } from 'apify-client/build/utils';
import * as _ from 'underscore';
import { ACTOR_EVENT_NAMES_EX } from './constants';
import Request from './request'; // eslint-disable-line import/no-duplicates
-import log from './utils_log';
import events from './events';
+import log from './utils_log';
import { getFirstKey, publicUtils } from './utils';
import { getValue, setValue } from './key_value_store';
import { serializeArray, createDeserialize } from './serialization';
@@ -18,7 +18,6 @@ export const REQUESTS_PERSISTENCE_KEY = 'REQUEST_LIST_REQUESTS';
const CONTENT_TYPE_BINARY = 'application/octet-stream';
-
/**
* @typedef RequestListOptions
* @property {Array<RequestOptions|Request|string>} [sources]
@@ -221,9 +220,11 @@ export class RequestList {
keepDuplicateUrls = false,
} = options;
+ this.log = log.child({ prefix: 'RequestList' });
+
// TODO Deprecated 02/2020
if (persistSourcesKey) {
- log.deprecated('RequestList: options.persistSourcesKey is deprecated. Use options.persistRequestsKey.');
+ this.log.deprecated('options.persistSourcesKey is deprecated. Use options.persistRequestsKey.');
}
checkParamOrThrow(sources, 'options.sources', 'Maybe Array');
@@ -235,7 +236,7 @@ export class RequestList {
checkParamOrThrow(keepDuplicateUrls, 'options.keepDuplicateUrls', 'Maybe Boolean');
if (!(sources || sourcesFunction)) {
- throw new Error('RequestList: At least one of "sources" or "sourcesFunction" must be provided.');
+ throw new Error('At least one of "sources" or "sourcesFunction" must be provided.');
}
// Array of all requests from all sources, in the order as they appeared in sources.
@@ -366,7 +367,7 @@ export class RequestList {
this._addRequest(source);
}
} catch (err) {
- throw new Error(`RequestList: Loading requests with sourcesFunction failed. Cause:\n${err.stack}`);
+ throw new Error(`Loading requests with sourcesFunction failed. Cause:\n${err.stack}`);
}
}
}
@@ -382,14 +383,14 @@ export class RequestList {
*/
async persistState() {
if (!this.persistStateKey) {
- throw new Error('RequestList: Cannot persist state. options.persistStateKey is not set.');
+ throw new Error('Cannot persist state. options.persistStateKey is not set.');
}
if (this.isStatePersisted) return;
try {
await setValue(this.persistStateKey, this.getState());
this.isStatePersisted = true;
} catch (err) {
- log.exception(err, 'RequestList attempted to persist state, but failed.');
+ this.log.exception(err, 'Attempted to persist state, but failed.');
}
}
@@ -421,18 +422,18 @@ export class RequestList {
throw new Error('The state object is invalid: nextIndex must be a non-negative number.');
}
if (state.nextIndex > this.requests.length) {
- throw new Error('The state object is not consistent with RequestList: too few requests loaded.');
+ throw new Error('The state object is not consistent with RequestList too few requests loaded.');
}
if (state.nextIndex < this.requests.length
&& this.requests[state.nextIndex].uniqueKey !== state.nextUniqueKey) {
- throw new Error('The state object is not consistent with RequestList: the order of URLs seems to have changed.');
+ throw new Error('The state object is not consistent with RequestList the order of URLs seems to have changed.');
}
const deleteFromInProgress = [];
_.keys(state.inProgress).forEach((uniqueKey) => {
const index = this.uniqueKeyToIndex[uniqueKey];
if (typeof index !== 'number') {
- throw new Error('The state object is not consistent with RequestList: unknown uniqueKey is present in the state.');
+ throw new Error('The state object is not consistent with RequestList. Unknown uniqueKey is present in the state.');
}
if (index >= state.nextIndex) {
deleteFromInProgress.push(uniqueKey);
@@ -453,7 +454,9 @@ export class RequestList {
// As a workaround, we just remove all inProgress requests whose index >= nextIndex,
// since they will be crawled again.
if (deleteFromInProgress.length) {
- log.warning('RequestList\'s in-progress field is not consistent, skipping invalid in-progress entries', { deleteFromInProgress });
+ this.log.warning('RequestList\'s in-progress field is not consistent, skipping invalid in-progress entries', {
+ deleteFromInProgress,
+ });
_.each(deleteFromInProgress, (uniqueKey) => {
delete state.inProgress[uniqueKey];
});
@@ -478,14 +481,14 @@ export class RequestList {
let persistedRequests;
if (this.initialState) {
state = this.initialState;
- log.debug('RequestList: Loaded state from options.state argument.');
+ this.log.debug('Loaded state from options.state argument.');
} else if (this.persistStateKey) {
state = getValue(this.persistStateKey);
- if (state) log.debug('RequestList: Loaded state from key value store using the persistStateKey.');
+ if (state) this.log.debug('Loaded state from key value store using the persistStateKey.');
}
if (this.persistRequestsKey) {
persistedRequests = await getValue(this.persistRequestsKey);
- if (persistedRequests) log.debug('RequestList: Loaded requests from key value store using the persistRequestsKey.');
+ if (persistedRequests) this.log.debug('Loaded requests from key value store using the persistRequestsKey.');
}
// Unwraps "state" promise if needed, otherwise no-op.
return Promise.all([state, persistedRequests]);
@@ -614,7 +617,7 @@ export class RequestList {
const fetchedCount = fetchedRequests.length;
const importedCount = this.requests.length - originalLength;
- log.info('RequestList: Fetched and loaded Requests from a remote resource.', {
+ this.log.info('Fetched and loaded Requests from a remote resource.', {
requestsFromUrl,
regex,
fetchedCount,
@@ -645,7 +648,7 @@ export class RequestList {
// Skip if resource contained no URLs.
if (!urlsArr.length) {
- log.warning('RequestList: list fetched, but it is empty.', { requestsFromUrl, regex });
+ this.log.warning('list fetched, but it is empty.', { requestsFromUrl, regex });
return [];
}
@@ -670,7 +673,7 @@ export class RequestList {
} else if (source && type === 'object') {
request = new Request(source);
} else {
- throw new Error(`RequestList: Cannot create Request from type: ${type}`);
+ throw new Error(`Cannot create Request from type: ${type}`);
}
const hasUniqueKey = !!source.uniqueKey;
@@ -688,7 +691,7 @@ export class RequestList {
this.uniqueKeyToIndex[uniqueKey] = this.requests.length;
this.requests.push(request);
} else if (this.keepDuplicateUrls) {
- log.warning(`RequestList: Duplicate uniqueKey: ${uniqueKey} found while the keepDuplicateUrls option was set. Check your sources' unique keys.`); // eslint-disable-line max-len
+ this.log.warning(`Duplicate uniqueKey: ${uniqueKey} found while the keepDuplicateUrls option was set. Check your sources' unique keys.`); // eslint-disable-line max-len
}
}
diff --git a/src/request_queue.js b/src/request_queue.js
index b6d833184..42a727676 100644
--- a/src/request_queue.js
+++ b/src/request_queue.js
@@ -6,9 +6,11 @@ import * as LruCache from 'apify-shared/lru_cache';
import * as ListDictionary from 'apify-shared/list_dictionary';
import { ENV_VARS, LOCAL_STORAGE_SUBDIRS, REQUEST_QUEUE_HEAD_MAX_LIMIT } from 'apify-shared/consts';
import { checkParamPrototypeOrThrow, cryptoRandomObjectId } from 'apify-shared/utilities';
-import log from './utils_log';
import Request, { RequestOptions } from './request'; // eslint-disable-line import/named,no-unused-vars
-import { ensureDirExists, apifyClient, openRemoteStorage, openLocalStorage, ensureTokenOrLocalStorageEnvExists, sleep } from './utils';
+import {
+ ensureDirExists, apifyClient, openRemoteStorage, openLocalStorage, ensureTokenOrLocalStorageEnvExists, sleep,
+} from './utils';
+import log from './utils_log';
export const LOCAL_STORAGE_SUBDIR = LOCAL_STORAGE_SUBDIRS.requestQueues;
const MAX_OPENED_QUEUES = 1000;
@@ -199,6 +201,7 @@ export class RequestQueue {
if (!clientKey) throw new Error('Parameter "clientKey" must be a non-empty string!');
+ this.log = log.child({ prefix: 'RequestQueue' });
this.clientKey = clientKey;
this.queueId = queueId;
this.queueName = queueName;
@@ -347,7 +350,7 @@ export class RequestQueue {
// This should never happen, but...
if (this.inProgress.has(nextRequestId) || this.recentlyHandled.get(nextRequestId)) {
- log.warning('Queue head returned a request that is already in progress?!', {
+ this.log.warning('Queue head returned a request that is already in progress?!', {
nextRequestId,
inProgress: this.inProgress.has(nextRequestId),
recentlyHandled: !!this.recentlyHandled.get(nextRequestId),
@@ -374,7 +377,7 @@ export class RequestQueue {
// into the queueHeadDict straight again. After the interval expires, fetchNextRequest()
// will try to fetch this request again, until it eventually appears in the main table.
if (!request) {
- log.debug('Cannot find a request from the beginning of queue, will be retried later', { nextRequestId });
+ this.log.debug('Cannot find a request from the beginning of queue, will be retried later', { nextRequestId });
setTimeout(() => {
this.inProgress.delete(nextRequestId);
}, STORAGE_CONSISTENCY_DELAY_MILLIS);
@@ -386,7 +389,7 @@ export class RequestQueue {
// We just add the request to the recentlyHandled dictionary so that next call to _ensureHeadIsNonEmpty()
// will not put the request again to queueHeadDict.
if (request.handledAt) {
- log.debug('Request fetched from the beginning of queue was already handled', { nextRequestId });
+ this.log.debug('Request fetched from the beginning of queue was already handled', { nextRequestId });
this.recentlyHandled.add(nextRequestId, true);
return null;
}
@@ -472,7 +475,7 @@ export class RequestQueue {
// This is to compensate for the limitation of DynamoDB, where writes might not be immediately visible to subsequent reads.
setTimeout(() => {
if (!this.inProgress.has(request.id)) {
- log.warning('The request is no longer marked as in progress in the queue?!', { requestId: request.id });
+ this.log.warning('The request is no longer marked as in progress in the queue?!', { requestId: request.id });
return;
}
@@ -596,7 +599,7 @@ export class RequestQueue {
// If limit was not reached in the call then there are no more requests to be returned.
if (prevLimit >= REQUEST_QUEUE_HEAD_MAX_LIMIT) {
- log.warning(`RequestQueue: Reached the maximum number of requests in progress: ${REQUEST_QUEUE_HEAD_MAX_LIMIT}.`);
+ this.log.warning(`Reached the maximum number of requests in progress: ${REQUEST_QUEUE_HEAD_MAX_LIMIT}.`);
}
const shouldRepeatWithHigherLimit = this.queueHeadDict.length() === 0
&& wasLimitReached
@@ -624,7 +627,7 @@ export class RequestQueue {
// If we are repeating for consistency then wait required time.
if (shouldRepeatForConsistency) {
const delayMillis = API_PROCESSED_REQUESTS_DELAY_MILLIS - (Date.now() - queueModifiedAt);
- log.info(`RequestQueue: Waiting for ${delayMillis}ms before considering the queue as finished to ensure that the data is consistent.`);
+ this.log.info(`Waiting for ${delayMillis}ms before considering the queue as finished to ensure that the data is consistent.`);
await sleep(delayMillis);
}
@@ -660,7 +663,7 @@ export class RequestQueue {
/** @ignore */
async delete() {
- log.deprecated('requestQueue.delete() is deprecated. Please use requestQueue.drop() instead. '
+ this.log.deprecated('requestQueue.delete() is deprecated. Please use requestQueue.drop() instead. '
+ 'This is to make it more obvious to users that the function deletes the request queue and not individual records in the queue.');
await this.drop();
}
@@ -741,6 +744,7 @@ export class RequestQueueLocal {
checkParamOrThrow(queueId, 'queueId', 'String');
checkParamOrThrow(localStorageDir, 'localStorageDir', 'String');
+ this.log = log.child({ prefix: 'RequestQueue' });
this.queueId = queueId;
this.localStoragePath = path.resolve(path.join(localStorageDir, LOCAL_STORAGE_SUBDIR, queueId));
this.localHandledEmulationPath = path.join(this.localStoragePath, 'handled');
@@ -1020,7 +1024,7 @@ export class RequestQueueLocal {
}
async delete() {
- log.deprecated('requestQueue.delete() is deprecated. Please use requestQueue.drop() instead. '
+ this.log.deprecated('requestQueue.delete() is deprecated. Please use requestQueue.drop() instead. '
+ 'This is to make it more obvious to users that the function deletes the request queue and not individual records in the queue.');
await this.drop();
}
diff --git a/src/session_pool/session.js b/src/session_pool/session.js
index b26f63a58..1cd260ddc 100644
--- a/src/session_pool/session.js
+++ b/src/session_pool/session.js
@@ -1,10 +1,10 @@
import { cryptoRandomObjectId } from 'apify-shared/utilities';
import { checkParamOrThrow } from 'apify-client/build/utils';
import { Cookie, CookieJar } from 'tough-cookie';
-import log from '../utils_log';
import EVENTS from './events';
import { STATUS_CODES_BLOCKED } from '../constants';
import { getCookiesFromResponse } from './session_utils';
+import defaultLog from '../utils_log';
// TYPE IMPORTS
/* eslint-disable no-unused-vars,import/named,import/no-duplicates,import/order,import/no-cycle */
@@ -77,6 +77,7 @@ export class Session {
errorScore = 0,
maxUsageCount = 50,
sessionPool,
+ log = defaultLog,
} = options;
const { expiresAt = this._getDefaultCookieExpirationDate(maxAgeSecs) } = options;
@@ -95,9 +96,11 @@ export class Session {
// sessionPool must be instance of SessionPool.
if (sessionPool.constructor.name !== 'SessionPool') {
- throw new Error('Session: sessionPool must be instance of SessionPool');
+ throw new Error('sessionPool must be instance of SessionPool');
}
+ this.log = log.child({ prefix: 'Session' });
+
/**
* @type {CookieJar}
* @private
@@ -243,7 +246,7 @@ export class Session {
this._setCookies(cookies, response.url);
} catch (e) {
// if invalid Cookie header is provided just log the exception.
- log.exception(e, 'Session: Could not get cookies from response');
+ this.log.exception(e, 'Could not get cookies from response');
}
}
@@ -268,7 +271,7 @@ export class Session {
this._setCookies(cookies.map(this._puppeteerCookieToTough.bind(this)), url);
} catch (e) {
// if invalid cookies are provided just log the exception. No need to retry the request automatically.
- log.exception(e, 'Session: Could not set cookies in puppeteer format.');
+ this.log.exception(e, 'Could not set cookies in puppeteer format.');
}
}
diff --git a/src/session_pool/session_pool.js b/src/session_pool/session_pool.js
index c4fb0dcba..3eef84101 100644
--- a/src/session_pool/session_pool.js
+++ b/src/session_pool/session_pool.js
@@ -1,10 +1,10 @@
import { EventEmitter } from 'events';
import { checkParamOrThrow } from 'apify-client/build/utils';
-import log from '../utils_log';
import { openKeyValueStore } from '../key_value_store';
import { Session, SessionOptions } from './session'; // eslint-disable-line no-unused-vars,import/named,import/no-cycle
import events from '../events';
+import defaultLog from '../utils_log';
import { ACTOR_EVENT_NAMES_EX } from '../constants';
/**
@@ -89,6 +89,7 @@ export class SessionPool extends EventEmitter {
createSessionFunction = null,
sessionOptions = {},
+ log = defaultLog,
} = options;
super();
@@ -100,12 +101,19 @@ export class SessionPool extends EventEmitter {
checkParamOrThrow(persistStateKey, 'options.persistStateKey', 'String');
checkParamOrThrow(createSessionFunction, 'options.createSessionFunction', 'Maybe Function');
+ this.log = log.child({ prefix: 'SessionPool' });
+
// Pool Configuration
this.maxPoolSize = maxPoolSize;
this.createSessionFunction = createSessionFunction || this._defaultCreateSessionFunction;
// Session configuration
- this.sessionOptions = sessionOptions;
+ this.sessionOptions = {
+ ...sessionOptions,
+ // the log needs to propagate to createSessionFunction as in "new Session({ ...sessionPool.sessionOptions })"
+ // and can't go inside _defaultCreateSessionFunction
+ log: this.log,
+ };
// Session keyValueStore
this.persistStateKeyValueStoreId = persistStateKeyValueStoreId;
@@ -191,7 +199,7 @@ export class SessionPool extends EventEmitter {
* @return {Promise<void>}
*/
async persistState() {
- log.debug('SessionPool: Persisting state',
+ this.log.debug('Persisting state',
{
persistStateKeyValueStoreId: this.persistStateKeyValueStoreId,
persistStateKey: this.persistStateKey,
@@ -217,7 +225,7 @@ export class SessionPool extends EventEmitter {
const sessionIndex = this.sessions.findIndex(storedSession => storedSession.id === session.id);
const removedSession = this.sessions.splice(sessionIndex, 1);
- log.debug(`SessionPool: Removed Session - ${removedSession.id}`);
+ this.log.debug(`Removed Session - ${removedSession.id}`);
}
/**
@@ -260,7 +268,7 @@ export class SessionPool extends EventEmitter {
const newSession = await this.createSessionFunction(this);
this._addSession(newSession);
- log.debug(`SessionPool: Created new Session - ${newSession.id}`);
+ this.log.debug(`Created new Session - ${newSession.id}`);
return newSession;
}
@@ -294,7 +302,7 @@ export class SessionPool extends EventEmitter {
if (!loadedSessionPool) return;
// Invalidate old sessions and load active sessions only
- log.debug('SessionPool: Recreating state from KeyValueStore',
+ this.log.debug('Recreating state from KeyValueStore',
{
persistStateKeyValueStoreId: this.persistStateKeyValueStoreId,
persistStateKey: this.persistStateKey,
@@ -310,7 +318,7 @@ export class SessionPool extends EventEmitter {
}
}
- log.debug(`SessionPool: ${this.usableSessionsCount} active sessions loaded from KeyValueStore`);
+ this.log.debug(`${this.usableSessionsCount} active sessions loaded from KeyValueStore`);
}
}
diff --git a/test/.eslintrc.json b/test/.eslintrc.json
index 1e6583889..c4cf79fbe 100644
--- a/test/.eslintrc.json
+++ b/test/.eslintrc.json
@@ -11,6 +11,7 @@
"afterEach": false,
"expect": false,
"test": false,
- "jest": false
+ "jest": false,
+ "jasmine": false
}
}
diff --git a/test/autoscaling/snapshotter.test.js b/test/autoscaling/snapshotter.test.js
index 84762e532..571f9a7cc 100644
--- a/test/autoscaling/snapshotter.test.js
+++ b/test/autoscaling/snapshotter.test.js
@@ -243,11 +243,11 @@ describe('Snapshotter', () => {
memCurrentBytes: toBytes(7500),
};
let logged = false;
- const warning = () => { logged = true; };
- const stub = sinon.stub(log, 'warning');
- stub.callsFake(warning);
process.env[ENV_VARS.MEMORY_MBYTES] = '10000';
const snapshotter = new Snapshotter({ maxUsedMemoryRatio: 0.5 });
+ const warning = () => { logged = true; };
+ const stub = sinon.stub(snapshotter.log, 'warning');
+ stub.callsFake(warning);
snapshotter._memoryOverloadWarning(memoryDataOverloaded);
expect(logged).toBe(true);
diff --git a/test/dataset.test.js b/test/dataset.test.js
index e2fe1492f..a368a98cb 100644
--- a/test/dataset.test.js
+++ b/test/dataset.test.js
@@ -521,8 +521,8 @@ describe('dataset', () => {
} catch (err) {
e = err;
}
- expect(e.message).toEqual('dataset.getData(): The response is too large for parsing. You can fix this by lowering the "limit" option.');
stub.restore();
+ expect(e.message).toEqual('dataset.getData(): The response is too large for parsing. You can fix this by lowering the "limit" option.'); // eslint-disable-line max-len
});
test('getInfo() should work', async () => {
diff --git a/test/puppeteer_pool.test.js b/test/puppeteer_pool.test.js
index 5f16110de..d71343d87 100644
--- a/test/puppeteer_pool.test.js
+++ b/test/puppeteer_pool.test.js
@@ -608,7 +608,7 @@ describe('PuppeteerPool', () => {
await pool._openNewTab(); // eslint-disable-line no-underscore-dangle
throw new Error('invalid error');
} catch (err) {
- expect(err.stack).toMatch('PuppeteerPool: browser.newPage() timed out.');
+ expect(err.stack).toMatch('browser.newPage() timed out.');
}
});
});
diff --git a/test/session_pool/session.test.js b/test/session_pool/session.test.js
index ecacd419a..744437e8f 100644
--- a/test/session_pool/session.test.js
+++ b/test/session_pool/session.test.js
@@ -37,7 +37,7 @@ describe('Session - testing session behaviour ', () => {
err = e;
}
expect(err).toBeDefined(); // eslint-disable-line
- expect(err.message.includes('Session: sessionPool must be instance of SessionPool')).toBe(true); // eslint-disable-line
+ expect(err.message.includes('sessionPool must be instance of SessionPool')).toBe(true); // eslint-disable-line
},
);
diff --git a/test/session_pool/session_pool.test.js b/test/session_pool/session_pool.test.js
index c52f7eda9..bb3535092 100644
--- a/test/session_pool/session_pool.test.js
+++ b/test/session_pool/session_pool.test.js
@@ -5,6 +5,7 @@ import events from '../../build/events';
import { ACTOR_EVENT_NAMES_EX } from '../../build/constants';
import { Session } from '../../build/session_pool/session';
import LocalStorageDirEmulator from '../local_storage_dir_emulator';
+import defaultLog from '../../build/utils_log';
describe('SessionPool - testing session pool', () => {
let sessionPool;
@@ -57,9 +58,12 @@ describe('SessionPool - testing session pool', () => {
await sessionPool.initialize();
await sessionPool.teardown();
- Object.entries(opts).forEach(([key, value]) => {
+ Object.entries(opts).filter(([key]) => key !== 'sessionOptions').forEach(([key, value]) => {
expect(sessionPool[key]).toEqual(value);
});
+ // log is appended to sessionOptions after sessionPool instantiation
+ expect(sessionPool.sessionOptions).toEqual({ ...opts.sessionOptions, log: jasmine.any(defaultLog.Log) });
+
const store = await Apify.openKeyValueStore('TEST');
await store.drop();
});
@@ -83,9 +87,12 @@ describe('SessionPool - testing session pool', () => {
await sessionPool.teardown();
- Object.entries(opts).forEach(([key, value]) => {
+ Object.entries(opts).filter(([key]) => key !== 'sessionOptions').forEach(([key, value]) => {
expect(sessionPool[key]).toEqual(value);
});
+ // log is appended to sessionOptions after sessionPool instantiation
+ expect(sessionPool.sessionOptions).toEqual({ ...opts.sessionOptions, log: jasmine.any(defaultLog.Log) });
+
const store = await Apify.openKeyValueStore('TEST');
await store.drop();
});
diff --git a/test/typescript/crawlers/basic_crawler.test.ts b/test/typescript/crawlers/basic_crawler.test.ts
index 224fce23b..9aa0e8a82 100644
--- a/test/typescript/crawlers/basic_crawler.test.ts
+++ b/test/typescript/crawlers/basic_crawler.test.ts
@@ -44,7 +44,7 @@ describe('BasicCrawler TS', () => {
}
}, { forefront: true });
- options.sessionPoolOptions!.sessionOptions!.userData!.userAgent === 'user-agent';
+ (options.sessionPoolOptions!.sessionOptions!.userData! as any).userAgent === 'user-agent';
})
});
});