Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Changing logging to plaintext and updating some messages (#67)

Marek Trunkát55ca0cefe399
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 00f44dba0..ad1d17171 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,6 +2,7 @@ xxxxxxxxxxxxxxxxxxx
===================
- `Apify.call()` now throws an `ApifyCallError` error if run doesn't succeed
- Renamed options `abortInstanceAfterRequestCount` of `PuppeteerPool` and `PuppeteerCrawler` to retireInstanceAfterRequestCcount
+- Logs are now in plain text instead of JSON for better readability.
0.5.22 / 2018-04-12
===================
diff --git a/package.json b/package.json
index 2d24e4227..d7d6b0fd0 100644
--- a/package.json
+++ b/package.json
@@ -45,7 +45,7 @@
},
"dependencies": {
"apify-client": "^0.2.7",
- "apify-shared": "^0.0.25",
+ "apify-shared": "^0.0.28",
"bluebird": "^3.5.0",
"content-type": "^1.0.3",
"fs-extra": "^5.0.0",
diff --git a/src/autoscaled_pool.js b/src/autoscaled_pool.js
index b5ced9067..d3b8e81c0 100644
--- a/src/autoscaled_pool.js
+++ b/src/autoscaled_pool.js
@@ -85,7 +85,7 @@ export default class AutoscaledPool {
if (opts.workerFunction) {
// For backwards compatiblity with opts.finishWhenEmpty and this.finish();
if (opts.finishWhenEmpty !== undefined) {
- log.warning('Parameter `finishWhenEmpty` of AutoscaledPool is deprecated!!! Use `isFinishedFunction` instead!');
+ log.warning('AutoscaledPool: Parameter `finishWhenEmpty` is deprecated!!! Use `isFinishedFunction` instead!');
checkParamOrThrow(opts.finishWhenEmpty, 'opts.finishWhenEmpty', 'Boolean');
let mayFinish = false;
opts.isFinishedFunction = () => Promise.resolve(mayFinish);
@@ -94,7 +94,7 @@ export default class AutoscaledPool {
opts.isFinishedFunction = () => Promise.resolve(true);
}
- log.warning('Parameter `workerFunction` of AutoscaledPool is deprecated!!! Use `runTaskFunction` instead!');
+ log.warning('AutoscaledPool: Parameter `workerFunction` is deprecated!!! Use `runTaskFunction` instead!');
checkParamOrThrow(opts.workerFunction, 'opts.workerFunction', 'Function');
opts.runTaskFunction = opts.workerFunction;
opts.isTaskReadyFunction = () => Promise.resolve(true);
diff --git a/src/basic_crawler.js b/src/basic_crawler.js
index b471db1f8..411acf6f7 100644
--- a/src/basic_crawler.js
+++ b/src/basic_crawler.js
@@ -8,7 +8,11 @@ import { RequestQueue, RequestQueueLocal } from './request_queue';
const DEFAULT_OPTIONS = {
maxRequestRetries: 3,
- handleFailedRequestFunction: ({ request }) => log.error('Request failed', _.pick(request, 'url', 'uniqueKey')),
+ handleFailedRequestFunction: ({ request }) => {
+ const details = _.pick(request, 'id', 'url', 'method', 'uniqueKey');
+
+ log.error('BasicCrawler: Request failed and reached maximum retries', details);
+ },
};
/**
@@ -179,7 +183,7 @@ export default class BasicCrawler {
// Retry request.
if (request.retryCount < this.maxRequestRetries) {
request.retryCount++;
- log.exception(error, 'Reclaiming failed request back to queue', {
+ log.exception(error, 'RequestQueue: handleRequestFunction failed, reclaiming failed request back to queue', {
url: request.url,
retryCount: request.retryCount,
});
@@ -187,7 +191,7 @@ export default class BasicCrawler {
return source.reclaimRequest(request);
}
- log.exception(error, 'Marking failed request handled', {
+ log.exception(error, 'RequestQueue: handleRequestFunction failed, marking failed request handled', {
url: request.url,
retryCount: request.retryCount,
});
diff --git a/src/events.js b/src/events.js
index 12db10c17..099c43591 100644
--- a/src/events.js
+++ b/src/events.js
@@ -124,7 +124,7 @@ export const initializeEvents = () => {
// Locally there is no web socket to connect so just print a log message.
if (!eventsWsUrl) {
- log.info(`Environment variable ${ENV_VARS.ACTOR_EVENTS_WS_URL} is not set, Apify.events will not emit any events.`);
+ log.info(`Apify.events: Environment variable ${ENV_VARS.ACTOR_EVENTS_WS_URL} is not set, no events from Apify platform will be emitted.`);
return;
}
@@ -142,12 +142,12 @@ export const initializeEvents = () => {
emitPersistStateEvent(true);
}
} catch (err) {
- log.exception(err, 'Cannot parse actor event');
+ log.exception(err, 'Apify.events: Cannot parse actor event');
}
});
- eventsWs.on('error', err => log.exception(err, 'Actor events web socket connection failed'));
+ eventsWs.on('error', err => log.exception(err, 'Apify.events: web socket connection failed'));
eventsWs.on('close', () => {
- log.warning('Actor events web socket has been closed');
+ log.warning('Apify.events: web socket has been closed');
eventsWs = null;
});
};
diff --git a/src/index.js b/src/index.js
index a01cf9570..97f318f06 100644
--- a/src/index.js
+++ b/src/index.js
@@ -21,6 +21,9 @@ import { browse, launchWebDriver } from './webdriver';
// Hide debug log messages when running in production mode.
if (!isProduction() || process.env[ENV_VARS.LOG_LEVEL] === 'DEBUG') log.isDebugMode = true;
+// Log as plain text not JSON
+log.logJson = false;
+
/**
*{include-readme-1}
*{include-readme-2}
diff --git a/src/puppeteer_crawler.js b/src/puppeteer_crawler.js
index 27fca7412..4003a977e 100644
--- a/src/puppeteer_crawler.js
+++ b/src/puppeteer_crawler.js
@@ -97,6 +97,7 @@ export default class PuppeteerCrawler {
constructor(opts) {
// For backwards compatibility, in the future we can remove this...
if (!opts.retireInstanceAfterRequestCount && opts.abortInstanceAfterRequestCount) {
+ log.warning('PuppeteerCrawler: Parameter `abortInstanceAfterRequestCount` is deprecated! Use `retireInstanceAfterRequestCount` instead!');
opts.retireInstanceAfterRequestCount = opts.abortInstanceAfterRequestCount;
}
diff --git a/src/puppeteer_pool.js b/src/puppeteer_pool.js
index f76106a67..de6f7ccc2 100644
--- a/src/puppeteer_pool.js
+++ b/src/puppeteer_pool.js
@@ -111,6 +111,7 @@ export default class PuppeteerPool {
// For backwards compatibility, in the future we can remove this...
if (!opts.retireInstanceAfterRequestCount && opts.abortInstanceAfterRequestCount) {
+ log.warning('PuppeteerPool: Parameter `abortInstanceAfterRequestCount` is deprecated! Use `retireInstanceAfterRequestCount` instead!');
opts.retireInstanceAfterRequestCount = opts.abortInstanceAfterRequestCount;
}
diff --git a/src/request_list.js b/src/request_list.js
index df6499712..f6398d7d1 100644
--- a/src/request_list.js
+++ b/src/request_list.js
@@ -192,7 +192,7 @@ export default class RequestList {
this.isStatePersisted = true;
})
.catch((err) => {
- log.exception(err, 'Cannot persist state of UrlList', { persistStateKey: this.persistStateKey });
+ log.exception(err, 'RequestList: Cannot persist state', { persistStateKey: this.persistStateKey });
});
});
});
diff --git a/test/actor.js b/test/actor.js
index fa3297f17..c8bede8ad 100644
--- a/test/actor.js
+++ b/test/actor.js
@@ -691,7 +691,6 @@ describe('Apify.call()', () => {
.call(actId, null, { token })
.then(() => { throw new Error('This was suppose to fail!'); }, (err) => {
expect(err).to.be.instanceOf(ApifyCallError);
- console.log(err);
expect(err.run.status).to.be.eql(ACT_TASK_STATUSES.ABORTED);
expect(err.run).to.be.eql(failedRun);
});