Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Adding function to create standardised debug info from request and response.

Marek Trunkata271d8d2a003
diff --git a/src/utils.js b/src/utils.js
index b522478b8..fd805414b 100644
--- a/src/utils.js
+++ b/src/utils.js
@@ -1,5 +1,6 @@
import contentTypeParser from 'content-type';
import os from 'os';
+import _ from 'underscore';
import fs from 'fs';
import util from 'util';
import fsExtra from 'fs-extra';
@@ -560,6 +561,35 @@ const htmlToText = (html) => {
return text.trim();
};
+/**
+ * Creates a standardized debug info from request and response. This info is usually added to dataset under the hidden `#debug` field.
+ *
+ * @param {Object} request [Apify.Request](https://sdk.apify.com/docs/api/request) object.
+ * @param {Object} [response] Puppeteer [Response](https://pptr.dev/#?product=Puppeteer&version=v1.11.0&show=api-class-response) object
+ * or NodeJS [http.ServerResponse](https://nodejs.org/api/http.html#http_class_http_serverresponse) object
+ * @param {Object} [additionalFields] Object containing additional fields to be added.
+
+ * @return {Object}
+ * @memberOf utils
+ */
+const createRequestDebugInfo = (request, response = {}, additionalFields = {}) => {
+ checkParamOrThrow(request, 'request', 'Object');
+ checkParamOrThrow(response, 'response', 'Object');
+ checkParamOrThrow(additionalFields, 'additionalFields', 'Object');
+
+ return Object.assign(
+ {
+ requestId: request.id,
+ url: request.url,
+ method: request.method,
+ retryCount: request.retryCount,
+ errorMessages: request.errorMessages,
+ // Puppeteer response has .status() funtion and NodeJS response ,statusCode property.
+ statusCode: _.isFunction(response.status) ? response.status() : response.statusCode,
+ },
+ additionalFields,
+ );
+};
/**
* A namespace that contains various utilities.
@@ -585,4 +615,5 @@ export const publicUtils = {
htmlToText,
URL_NO_COMMAS_REGEX,
URL_WITH_COMMAS_REGEX,
+ createRequestDebugInfo,
};
diff --git a/test/utils.js b/test/utils.js
index 6fe0cd72b..d3e95426e 100644
--- a/test/utils.js
+++ b/test/utils.js
@@ -752,25 +752,25 @@ describe('utils.htmlToText()', () => {
/* eslint-disable no-tabs */
checkHtmlToText(
html1,
- `Let's start with a simple text.
-The ships hung in the sky, much the way that bricks don't.
+ `Let's start with a simple text.
+The ships hung in the sky, much the way that bricks don't.
These aren't the Droids you're looking for
I'm sorry, Dave. I'm afraid I can't do that.
I'm sorry, Dave. I'm afraid I can't do that.
-A1 A2 A3
-B1 B2 B3 B 4
-This is some text with inline elements and HTML entities (>bla<)
+A1 A2 A3
+B1 B2 B3 B 4
+This is some text with inline elements and HTML entities (>bla<)
Test
a
few
line
breaks
-Spaces in an inline text should be completely ignored.
+Spaces in an inline text should be completely ignored.
But,
a pre-formatted
block should be kept
pre-formatted.
-The Greatest Science Fiction Quotes Of All Time
+The Greatest Science Fiction Quotes Of All Time
Don't know, I don't know such stuff. I just do eyes, ju-, ju-, just eyes... just genetic design, just eyes. You Nexus, huh? I design your eyes.`,
true,
);
@@ -785,3 +785,68 @@ Don't know, I don't know such stuff. I just do eyes, ju-, ju-, just eyes... just
checkHtmlToText(cheerio.load(html2, { decodeEntities: true }), 'Text outside of body');
});
});
+
+
+describe('utils.createRequestDebugInfo()', () => {
+ it('handles Puppeteer response', () => {
+ const request = {
+ id: 'some-id',
+ url: 'https://example.com',
+ method: 'POST',
+ retryCount: 2,
+ errorMessages: ['xxx'],
+ someThingElse: 'xxx',
+ someOther: 'yyy',
+ };
+
+ const response = {
+ status: () => 201,
+ another: 'yyy',
+ };
+
+ const additionalFields = {
+ foo: 'bar',
+ };
+
+ expect(Apify.utils.createRequestDebugInfo(request, response, additionalFields)).to.be.eql({
+ requestId: 'some-id',
+ url: 'https://example.com',
+ method: 'POST',
+ retryCount: 2,
+ errorMessages: ['xxx'],
+ statusCode: 201,
+ foo: 'bar',
+ });
+ });
+
+ it('handles NodeJS response', () => {
+ const request = {
+ id: 'some-id',
+ url: 'https://example.com',
+ method: 'POST',
+ retryCount: 2,
+ errorMessages: ['xxx'],
+ someThingElse: 'xxx',
+ someOther: 'yyy',
+ };
+
+ const response = {
+ statusCode: 201,
+ another: 'yyy',
+ };
+
+ const additionalFields = {
+ foo: 'bar',
+ };
+
+ expect(Apify.utils.createRequestDebugInfo(request, response, additionalFields)).to.be.eql({
+ requestId: 'some-id',
+ url: 'https://example.com',
+ method: 'POST',
+ retryCount: 2,
+ errorMessages: ['xxx'],
+ statusCode: 201,
+ foo: 'bar',
+ });
+ });
+});