Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Adding function to create standardised debug info from request and response.
Marek Trunkata271d8d2a003
diff --git a/src/utils.js b/src/utils.jsindex b522478b8..fd805414b 100644--- a/src/utils.js+++ b/src/utils.js@@ -1,5 +1,6 @@import contentTypeParser from 'content-type';import os from 'os';+import _ from 'underscore';import fs from 'fs';import util from 'util';import fsExtra from 'fs-extra';@@ -560,6 +561,35 @@ const htmlToText = (html) => {return text.trim();};+/**+ * Creates a standardized debug info from request and response. This info is usually added to dataset under the hidden `#debug` field.+ *+ * @param {Object} request [Apify.Request](https://sdk.apify.com/docs/api/request) object.+ * @param {Object} [response] Puppeteer [Response](https://pptr.dev/#?product=Puppeteer&version=v1.11.0&show=api-class-response) object+ * or NodeJS [http.ServerResponse](https://nodejs.org/api/http.html#http_class_http_serverresponse) object+ * @param {Object} [additionalFields] Object containing additional fields to be added.++ * @return {Object}+ * @memberOf utils+ */+const createRequestDebugInfo = (request, response = {}, additionalFields = {}) => {+ checkParamOrThrow(request, 'request', 'Object');+ checkParamOrThrow(response, 'response', 'Object');+ checkParamOrThrow(additionalFields, 'additionalFields', 'Object');++ return Object.assign(+ {+ requestId: request.id,+ url: request.url,+ method: request.method,+ retryCount: request.retryCount,+ errorMessages: request.errorMessages,+ // Puppeteer response has .status() funtion and NodeJS response ,statusCode property.+ statusCode: _.isFunction(response.status) ? response.status() : response.statusCode,+ },+ additionalFields,+ );+};/*** A namespace that contains various utilities.@@ -585,4 +615,5 @@ export const publicUtils = {htmlToText,URL_NO_COMMAS_REGEX,URL_WITH_COMMAS_REGEX,+ createRequestDebugInfo,};diff --git a/test/utils.js b/test/utils.jsindex 6fe0cd72b..d3e95426e 100644--- a/test/utils.js+++ b/test/utils.js@@ -752,25 +752,25 @@ describe('utils.htmlToText()', () => {/* eslint-disable no-tabs */checkHtmlToText(html1,- `Let's start with a simple text.-The ships hung in the sky, much the way that bricks don't.+ `Let's start with a simple text.+The ships hung in the sky, much the way that bricks don't.These aren't the Droids you're looking forI'm sorry, Dave. I'm afraid I can't do that.I'm sorry, Dave. I'm afraid I can't do that.-A1 A2 A3-B1 B2 B3 B 4-This is some text with inline elements and HTML entities (>bla<)+A1 A2 A3+B1 B2 B3 B 4+This is some text with inline elements and HTML entities (>bla<)Testafewlinebreaks-Spaces in an inline text should be completely ignored.+Spaces in an inline text should be completely ignored.But,a pre-formattedblock should be keptpre-formatted.-The Greatest Science Fiction Quotes Of All Time+The Greatest Science Fiction Quotes Of All TimeDon't know, I don't know such stuff. I just do eyes, ju-, ju-, just eyes... just genetic design, just eyes. You Nexus, huh? I design your eyes.`,true,);@@ -785,3 +785,68 @@ Don't know, I don't know such stuff. I just do eyes, ju-, ju-, just eyes... justcheckHtmlToText(cheerio.load(html2, { decodeEntities: true }), 'Text outside of body');});});+++describe('utils.createRequestDebugInfo()', () => {+ it('handles Puppeteer response', () => {+ const request = {+ id: 'some-id',+ url: 'https://example.com',+ method: 'POST',+ retryCount: 2,+ errorMessages: ['xxx'],+ someThingElse: 'xxx',+ someOther: 'yyy',+ };++ const response = {+ status: () => 201,+ another: 'yyy',+ };++ const additionalFields = {+ foo: 'bar',+ };++ expect(Apify.utils.createRequestDebugInfo(request, response, additionalFields)).to.be.eql({+ requestId: 'some-id',+ url: 'https://example.com',+ method: 'POST',+ retryCount: 2,+ errorMessages: ['xxx'],+ statusCode: 201,+ foo: 'bar',+ });+ });++ it('handles NodeJS response', () => {+ const request = {+ id: 'some-id',+ url: 'https://example.com',+ method: 'POST',+ retryCount: 2,+ errorMessages: ['xxx'],+ someThingElse: 'xxx',+ someOther: 'yyy',+ };++ const response = {+ statusCode: 201,+ another: 'yyy',+ };++ const additionalFields = {+ foo: 'bar',+ };++ expect(Apify.utils.createRequestDebugInfo(request, response, additionalFields)).to.be.eql({+ requestId: 'some-id',+ url: 'https://example.com',+ method: 'POST',+ retryCount: 2,+ errorMessages: ['xxx'],+ statusCode: 201,+ foo: 'bar',+ });+ });+});