Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
refactor: move buffer and stream helpers to `@crawlee/core` (#3928)
Jindřich Bär3ac7fc41c66e
diff --git a/packages/core/src/autoscaling/load_signal.ts b/packages/core/src/autoscaling/load_signal.tsindex cfea06546..11d8228fc 100644--- a/packages/core/src/autoscaling/load_signal.ts+++ b/packages/core/src/autoscaling/load_signal.ts@@ -1,6 +1,5 @@-import { weightedAvg } from '@crawlee/utils';-import type { ClientInfo } from './system_status.js';+import { weightedAvg } from './weighted_avg.js';/*** A snapshot of a resource's overload state at a point in time.diff --git a/packages/core/src/autoscaling/weighted_avg.ts b/packages/core/src/autoscaling/weighted_avg.tsnew file mode 100644index 000000000..a6953a6c0--- /dev/null+++ b/packages/core/src/autoscaling/weighted_avg.ts@@ -0,0 +1,16 @@+/**+ * Computes a weighted average of an array of numbers, complemented by an array of weights.+ * @ignore+ */+export function weightedAvg(arrValues: number[], arrWeights: number[]): number {+ const result = arrValues+ .map((value, i) => {+ const weight = arrWeights[i];+ const sum = value * weight;++ return [sum, weight];+ })+ .reduce((p, c) => [p[0] + c[0], p[1] + c[1]], [0, 0]);++ return result[0] / result[1];+}diff --git a/packages/core/src/byte_utils.ts b/packages/core/src/byte_utils.tsnew file mode 100644index 000000000..5c4afb608--- /dev/null+++ b/packages/core/src/byte_utils.ts@@ -0,0 +1,51 @@+/**+ * Checks if the given value is a Node.js Stream or a Web API ReadableStream.+ * @ignore+ */+export function isStream(value: unknown): value is NodeJS.ReadableStream | ReadableStream {+ if (typeof value !== 'object' || value === null) {+ return false;+ }++ // A Node.js Readable is both pipeable and async-iterable; a Web ReadableStream exposes pipeTo.+ // Requiring async-iterability for the `pipe` branch rejects plain `{ pipe }` ducks that would+ // otherwise blow up later in the storage backends' drain loop with a cryptic TypeError.+ const isNodeStream =+ typeof (value as any).pipe === 'function' && typeof (value as any)[Symbol.asyncIterator] === 'function';+ const isWebStream = typeof (value as any).pipeTo === 'function';++ return isNodeStream || isWebStream;+}++/**+ * Checks if the given value is a Node.js Buffer, ArrayBuffer, or TypedArray.+ * @ignore+ */+export function isBuffer(value: unknown): value is Buffer | ArrayBuffer | ArrayBufferView {+ return (+ value != null &&+ typeof value === 'object' &&+ (Buffer.isBuffer(value) ||+ value instanceof ArrayBuffer ||+ ArrayBuffer.isView(value) ||+ (value as any).constructor?.name === 'Buffer')+ );+}++/**+ * Converts a byte-like value (Buffer, ArrayBuffer, or any typed-array / DataView) into a Buffer over+ * the exact same bytes, honoring `byteOffset` / `byteLength` for views. Existing Buffers are returned+ * as-is. Used by storage backends, which persist raw bytes regardless of the input's concrete shape.+ * @ignore+ */+export function toBuffer(value: Buffer | ArrayBuffer | ArrayBufferView): Buffer {+ if (Buffer.isBuffer(value)) {+ return value;+ }++ if (value instanceof ArrayBuffer) {+ return Buffer.from(value);+ }++ return Buffer.from(value.buffer, value.byteOffset, value.byteLength);+}diff --git a/packages/core/src/memory-storage/utils.ts b/packages/core/src/memory-storage/utils.tsindex 36cded92e..a87dddd6c 100644--- a/packages/core/src/memory-storage/utils.ts+++ b/packages/core/src/memory-storage/utils.ts@@ -1,8 +1,6 @@import { createHash } from 'node:crypto';import { resolve, sep } from 'node:path';-import { isBuffer, isStream, toBuffer } from '@crawlee/utils';-import { REQUEST_ID_LENGTH } from './consts.js';/**@@ -50,4 +48,4 @@ export function uniqueKeyToRequestId(uniqueKey: string): string {return str.length > REQUEST_ID_LENGTH ? str.slice(0, REQUEST_ID_LENGTH) : str;}-export { isBuffer, isStream, toBuffer };+export { isBuffer, isStream, toBuffer } from '../byte_utils.js';diff --git a/packages/core/src/storages/key_value_store.ts b/packages/core/src/storages/key_value_store.tsindex 8428033aa..13369d624 100644--- a/packages/core/src/storages/key_value_store.ts+++ b/packages/core/src/storages/key_value_store.ts@@ -19,7 +19,7 @@ import type { StorageOpenOptions } from './utils.js';import type { StorageIdentifier } from './storage_instance_manager.js';import { resolveStorageIdentifier } from './storage_instance_manager.js';import { createDualIterable, purgeDefaultStorages } from './utils.js';-import { isBuffer, isStream } from '@crawlee/utils';+import { isBuffer, isStream } from '../byte_utils.js';/** @internal */const KVS_KEYS_DEFAULT_LIMIT = 1000;diff --git a/packages/core/src/storages/key_value_store_codec.ts b/packages/core/src/storages/key_value_store_codec.tsindex d17747af4..0479e6663 100644--- a/packages/core/src/storages/key_value_store_codec.ts+++ b/packages/core/src/storages/key_value_store_codec.ts@@ -1,10 +1,11 @@import type { Dictionary } from '@crawlee/types';import contentTypeParser from 'content-type';-import { isBuffer, isStream } from '@crawlee/utils';import JSON5 from 'json5';import { jsonStringifyExtended } from '@apify/utilities';+import { isBuffer, isStream } from '../byte_utils.js';+const CONTENT_TYPE_JSON = 'application/json';const STRINGIFIABLE_CONTENT_TYPE_RXS = [new RegExp(`^${CONTENT_TYPE_JSON}$`, 'i'), /^application\/.*xml$/i, /^text\//i];diff --git a/packages/utils/src/internals/general.ts b/packages/utils/src/internals/general.tsindex 975f326f3..5099ead40 100644--- a/packages/utils/src/internals/general.ts+++ b/packages/utils/src/internals/general.ts@@ -14,23 +14,6 @@ export const URL_NO_COMMAS_REGEX =export const URL_WITH_COMMAS_REGEX =/https?:\/\/(www\.)?([\p{L}0-9]|[\p{L}0-9][-\p{L}0-9@:%._+~#=]{0,254}[\p{L}0-9])\.[a-z]{2,63}(:\d{1,5})?(\/[-\p{L}0-9@:%_+,.~#?&/=()'*]*)?/giu;-/**- * Computes a weighted average of an array of numbers, complemented by an array of weights.- * @ignore- */-export function weightedAvg(arrValues: number[], arrWeights: number[]): number {- const result = arrValues- .map((value, i) => {- const weight = arrWeights[i];- const sum = value * weight;-- return [sum, weight];- })- .reduce((p, c) => [p[0] + c[0], p[1] + c[1]], [0, 0]);-- return result[0] / result[1];-}-/*** Returns a `Promise` that resolves after a specific period of time. This is useful to implement waiting* in your code, e.g. to prevent overloading of target website or to avoid bot detection.@@ -86,55 +69,3 @@ export function expandShadowRoots(document: Document): string {return document.documentElement.outerHTML;}--/**- * Checks if the given value is a Node.js Stream or a Web API ReadableStream.- * @ignore- */-export function isStream(value: unknown): value is NodeJS.ReadableStream | ReadableStream {- if (typeof value !== 'object' || value === null) {- return false;- }-- // A Node.js Readable is both pipeable and async-iterable; a Web ReadableStream exposes pipeTo.- // Requiring async-iterability for the `pipe` branch rejects plain `{ pipe }` ducks that would- // otherwise blow up later in the storage backends' drain loop with a cryptic TypeError.- const isNodeStream =- typeof (value as any).pipe === 'function' && typeof (value as any)[Symbol.asyncIterator] === 'function';- const isWebStream = typeof (value as any).pipeTo === 'function';-- return isNodeStream || isWebStream;-}--/**- * Checks if the given value is a Node.js Buffer, ArrayBuffer, or TypedArray.- * @ignore- */-export function isBuffer(value: unknown): value is Buffer | ArrayBuffer | ArrayBufferView {- return (- value != null &&- typeof value === 'object' &&- (Buffer.isBuffer(value) ||- value instanceof ArrayBuffer ||- ArrayBuffer.isView(value) ||- (value as any).constructor?.name === 'Buffer')- );-}--/**- * Converts a byte-like value (Buffer, ArrayBuffer, or any typed-array / DataView) into a Buffer over- * the exact same bytes, honoring `byteOffset` / `byteLength` for views. Existing Buffers are returned- * as-is. Used by storage backends, which persist raw bytes regardless of the input's concrete shape.- * @ignore- */-export function toBuffer(value: Buffer | ArrayBuffer | ArrayBufferView): Buffer {- if (Buffer.isBuffer(value)) {- return value;- }-- if (value instanceof ArrayBuffer) {- return Buffer.from(value);- }-- return Buffer.from(value.buffer, value.byteOffset, value.byteLength);-}diff --git a/test/core/autoscaling/weighted_avg.test.ts b/test/core/autoscaling/weighted_avg.test.tsnew file mode 100644index 000000000..46a647981--- /dev/null+++ b/test/core/autoscaling/weighted_avg.test.ts@@ -0,0 +1,13 @@+import { weightedAvg } from '../../../packages/core/src/autoscaling/weighted_avg.js';++describe('weightedAvg()', () => {+ test('works', () => {+ expect(weightedAvg([10, 10, 10], [1, 1, 1])).toBe(10);+ expect(weightedAvg([5, 10, 15], [1, 1, 1])).toBe(10);+ expect(weightedAvg([10, 10, 10], [0.5, 1, 1.5])).toBe(10);+ expect(weightedAvg([29, 35, 89], [13, 91, 3])).toEqual((29 * 13 + 35 * 91 + 89 * 3) / (13 + 91 + 3));+ expect(weightedAvg([], [])).toEqual(NaN);+ expect(weightedAvg([1], [0])).toEqual(NaN);+ expect(weightedAvg([], [1])).toEqual(NaN);+ });+});diff --git a/test/core/storages/key_value_store.test.ts b/test/core/storages/key_value_store.test.tsindex f43bdbfe0..ea171c0d8 100644--- a/test/core/storages/key_value_store.test.ts+++ b/test/core/storages/key_value_store.test.ts@@ -2,7 +2,8 @@ import { PassThrough } from 'node:stream';import { KeyValueStore, MemoryStorageBackend, serviceLocator } from '@crawlee/core';import type { Dictionary } from '@crawlee/types';-import { toBuffer } from '@crawlee/utils';++import { toBuffer } from '../../../packages/core/src/byte_utils.js';beforeEach(async () => {serviceLocator.setStorageBackend(new MemoryStorageBackend());diff --git a/test/utils/general.test.ts b/test/utils/general.test.tsindex 8db466708..32dfa1818 100644--- a/test/utils/general.test.ts+++ b/test/utils/general.test.ts@@ -1,16 +1,4 @@-import { sleep, weightedAvg } from '@crawlee/utils';--describe('weightedAvg()', () => {- test('works', () => {- expect(weightedAvg([10, 10, 10], [1, 1, 1])).toBe(10);- expect(weightedAvg([5, 10, 15], [1, 1, 1])).toBe(10);- expect(weightedAvg([10, 10, 10], [0.5, 1, 1.5])).toBe(10);- expect(weightedAvg([29, 35, 89], [13, 91, 3])).toEqual((29 * 13 + 35 * 91 + 89 * 3) / (13 + 91 + 3));- expect(weightedAvg([], [])).toEqual(NaN);- expect(weightedAvg([1], [0])).toEqual(NaN);- expect(weightedAvg([], [1])).toEqual(NaN);- });-});+import { sleep } from '@crawlee/utils';describe('sleep()', () => {test('works', async () => {