Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
refactor: move debug helpers to `@crawlee/core` (#3916)
Jindřich Bär39f626d64436
diff --git a/docs/public-api/crawlee-core.api.md b/docs/public-api/crawlee-core.api.mdindex 53e1620ee..2820f4e36 100644--- a/docs/public-api/crawlee-core.api.md+++ b/docs/public-api/crawlee-core.api.md@@ -21,6 +21,7 @@ import type { DatasetBackend } from '@crawlee/types';import type { DatasetInfo } from '@crawlee/types';import { Dictionary } from '@crawlee/types';import type { HttpRequestOptions } from '@crawlee/types';+import type { IncomingMessage } from 'node:http';import type { ISession } from '@crawlee/types';import type { ISessionPool } from '@crawlee/types';import type { KeyValueStoreBackend } from '@crawlee/types';diff --git a/docs/public-api/crawlee-utils.api.md b/docs/public-api/crawlee-utils.api.mdindex dbd5b131f..8536945df 100644--- a/docs/public-api/crawlee-utils.api.md+++ b/docs/public-api/crawlee-utils.api.md@@ -4,7 +4,6 @@```ts-import type { AllowedHttpMethods } from '@crawlee/types';import { Awaitable } from '@crawlee/types';import type { BaseHttpClient } from '@crawlee/types';import { Cheerio } from 'cheerio';@@ -13,7 +12,6 @@ import { Constructor } from '@crawlee/types';import type { CrawleeLogger } from '@crawlee/types';import { Dictionary } from '@crawlee/types';import { Element as Element_2 } from 'domhandler';-import type { IncomingMessage } from 'node:http';import type { SearchParams } from '@crawlee/types';export { Awaitable }@@ -33,9 +31,6 @@ export const CLOUDFLARE_RETRY_CSS_SELECTORS: string[];export { Constructor }-// @public-export function createRequestDebugInfo(request: Request_2, response?: IncomingMessage | Partial<BrowserResponseLike>, additionalFields?: Dictionary): Dictionary;-export { Dictionary }// @public@@ -98,9 +93,6 @@ const FACEBOOK_REGEX: RegExp;// @publicconst FACEBOOK_REGEX_GLOBAL: RegExp;-// @public (undocumented)-export function getObjectType(value: unknown): string;-// @publicexport function htmlToText(htmlOrCheerioElement: string | CheerioRoot): Promise<string>;diff --git a/packages/basic-crawler/src/internals/basic-crawler.ts b/packages/basic-crawler/src/internals/basic-crawler.tsindex 1b6f665db..edc07ee3c 100644--- a/packages/basic-crawler/src/internals/basic-crawler.ts+++ b/packages/basic-crawler/src/internals/basic-crawler.ts@@ -41,6 +41,7 @@ import {enqueueLinks,EnqueueStrategy,EventType,+ getObjectType,KeyValueStore,log,LogLevel,@@ -77,7 +78,7 @@ import type {SetStatusMessageOptions,StorageBackend,} from '@crawlee/types';-import { getObjectType, isAsyncIterable, isIterable, RobotsTxtFile, ROTATE_PROXY_ERRORS } from '@crawlee/utils';+import { isAsyncIterable, isIterable, RobotsTxtFile, ROTATE_PROXY_ERRORS } from '@crawlee/utils';import { stringify } from 'csv-stringify/sync';import { ensureDir, writeJSON } from 'fs-extra/esm';import ow from 'ow';diff --git a/packages/utils/src/internals/debug.ts b/packages/core/src/debug.tssimilarity index 85%rename from packages/utils/src/internals/debug.tsrename to packages/core/src/debug.tsindex dbb8efb49..37a9f1b00 100644--- a/packages/utils/src/internals/debug.ts+++ b/packages/core/src/debug.ts@@ -1,28 +1,15 @@import type { IncomingMessage } from 'node:http';import { inspect } from 'node:util';-import type { AllowedHttpMethods, Dictionary } from '@crawlee/types';+import type { Dictionary } from '@crawlee/types';import ow from 'ow';+import type { Request } from './request.js';+interface BrowserResponseLike {status(): number;}-interface Request<UserData extends Dictionary = Dictionary> {- id?: string;- url: string;- loadedUrl?: string;- uniqueKey: string;- method: AllowedHttpMethods;- payload?: string;- noRetry: boolean;- retryCount: number;- errorMessages: string[];- headers?: Record<string, string>;- userData: UserData;- handledAt?: string;-}-/*** Creates a standardized debug info from request and response. This info is usually added to dataset under the hidden `#debug` field.*@@ -31,6 +18,8 @@ interface Request<UserData extends Dictionary = Dictionary> {* Puppeteer [`Response`](https://pptr.dev/#?product=Puppeteer&version=v1.11.0&show=api-class-response)* or NodeJS [`http.IncomingMessage`](https://nodejs.org/api/http.html#http_class_http_serverresponse).* @param [additionalFields] Object containing additional fields to be added.+ *+ * @internal*/export function createRequestDebugInfo(request: Request,@@ -80,6 +69,12 @@ export function inspectValue(value: unknown): string {});}+/**+ * Returns the type of a value as a lowercase string, with `Date`, `Buffer` and `RegExp` reported+ * by their constructor name. Used for building validation error messages.+ *+ * @internal+ */export function getObjectType(value: unknown): string {const simple = typeof value;diff --git a/packages/core/src/errors.ts b/packages/core/src/errors.tsindex 92a30dbe6..13a1c79b3 100644--- a/packages/core/src/errors.ts+++ b/packages/core/src/errors.ts@@ -1,4 +1,4 @@-import { inspectValue } from '@crawlee/utils';+import { inspectValue } from './debug.js';/*** Errors of `NonRetryableError` type will never be retried by the crawler.diff --git a/packages/core/src/index.ts b/packages/core/src/index.tsindex bff597a44..e1cc54c27 100644--- a/packages/core/src/index.ts+++ b/packages/core/src/index.ts@@ -1,3 +1,4 @@+export * from './debug.js';export * from './errors.js';export * from './autoscaling/index.js';export * from './configuration.js';diff --git a/packages/core/src/storages/request_queue.ts b/packages/core/src/storages/request_queue.tsindex 5366e1c34..6510fb561 100644--- a/packages/core/src/storages/request_queue.ts+++ b/packages/core/src/storages/request_queue.ts@@ -12,7 +12,6 @@ import type {import {chunkedAsyncIterable,downloadListOfUrls,- getObjectType,isAsyncIterable,isIterable,peekableAsyncIterable,@@ -24,6 +23,7 @@ import type { ReadonlyDeep } from 'type-fest';import { LruCache } from '@apify/datastructures';import { Configuration } from '../configuration.js';+import { getObjectType } from '../debug.js';import type { Constructor } from '../typedefs.js';import type { EventManager } from '../events/event_manager.js';import { EventType } from '../events/event_manager.js';diff --git a/packages/utils/src/index.ts b/packages/utils/src/index.tsindex 517c14e6c..b58e56dcf 100644--- a/packages/utils/src/index.ts+++ b/packages/utils/src/index.ts@@ -3,7 +3,6 @@ export * from './internals/cheerio.js';export * from './internals/chunk.js';export * from './internals/extract-urls.js';export * from './internals/general.js';-export * from './internals/debug.js';export * as social from './internals/social.js';export * from './internals/typedefs.js';export * from './internals/open_graph_parser.js';